Merged in code from Arq 2.

AppKeychain.h

@@ -1,17 +1,27 @@
 //
 //  AppKeychain.h
-//  arq_restore
+//  Backup
 //
-//  Created by Stefan Reitshamer on 8/19/10.
+//  Created by Stefan Reitshamer on 8/26/09.
-//  Copyright 2010 __MyCompanyName__. All rights reserved.
+//  Copyright 2009 PhotoMinds LLC. All rights reserved.
 //
 
 #import <Cocoa/Cocoa.h>
 
 
 @interface AppKeychain : NSObject {
+    NSString *backupAppPath;
+    NSString *agentAppPath;
+    SecAccessRef access;
 }
-+ (NSString *)errorDomain;
 + (BOOL)accessKeyID:(NSString **)accessKeyID secretAccessKey:(NSString **)secret error:(NSError **)error;
-+ (BOOL)encryptionKey:(NSString **)encryptionKey error:(NSError **)error;
++ (BOOL)containsEncryptionPasswordForS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID;
++ (BOOL)encryptionPassword:(NSString **)encryptionPassword forS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID error:(NSError **)error;
++ (BOOL)twitterAccessKey:(NSString **)theAccessKey secret:(NSString **)secret error:(NSError **)error;
+
+- (id)initWithBackupAppPath:(NSString *)backupAppPath agentAppPath:(NSString *)agentAppPath;
+- (BOOL)setAccessKeyID:(NSString *)theAccessKeyID secretAccessKey:(NSString *)theSecretAccessKey error:(NSError **)error;
+- (BOOL)setEncryptionKey:(NSString *)theEncryptionPassword forS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID error:(NSError **)error;
+- (BOOL)setTwitterAccessKey:(NSString *)theKey secret:(NSString *)theSecret error:(NSError **)error;
+- (BOOL)deleteTwitterAccessKey:(NSError **)error;
 @end

AppKeychain.m

@@ -1,43 +1,271 @@
 //
 //  AppKeychain.m
-//  arq_restore
+//  Backup
 //
-//  Created by Stefan Reitshamer on 8/19/10.
+//  Created by Stefan Reitshamer on 8/26/09.
-//  Copyright 2010 __MyCompanyName__. All rights reserved.
+//  Copyright 2009 PhotoMinds LLC. All rights reserved.
 //
 
 #import "AppKeychain.h"
 #import "SetNSError.h"
-#import "NSErrorCodes.h"
+#import "UserLibrary.h"
+
+#define ARQ_S3_LABEL @"Arq S3"
+#define ARQ_TWITTER_LABEL @"Arq Twitter Access Token"
+#define ARQ_ENCRYPTION_ACCOUNT_LABEL @"EncryptionKey"
+
+@interface AppKeychain (internal)
++ (NSString *)encryptionPasswordLabelForS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID;
++ (BOOL)account:(NSString **)account password:(NSString **)password forLabel:(NSString *)label error:(NSError **)error;
++ (BOOL)findItem:(NSString *)theLabel item:(SecKeychainItemRef *)item error:(NSError **)error;
+- (BOOL)loadSecAccess:(NSError **)error;
+- (BOOL)deleteLabel:(NSString *)theLabel error:(NSError **)error;
+- (BOOL)addLabel:(NSString *)theLabel account:(NSString *)theAccount password:(NSString *)thePassword error:(NSError **)error;
+@end
 
 @implementation AppKeychain
-+ (NSString *)errorDomain {
++ (BOOL)accessKeyID:(NSString **)accessKey secretAccessKey:(NSString **)secret error:(NSError **)error {
-    return @"AppKeychainErrorDomain";
+    NSString *account = nil;
-}
+    NSString *password = nil;
-+ (BOOL)accessKeyID:(NSString **)accessKeyID secretAccessKey:(NSString **)secret error:(NSError **)error {
+    if (![AppKeychain account:&account password:&password forLabel:ARQ_S3_LABEL error:error]) {
-    char *cAccessKey = getenv("ARQ_ACCESS_KEY");
-    if (cAccessKey == NULL) {
-        SETNSERROR([AppKeychain errorDomain], ERROR_NOT_FOUND, @"ARQ_ACCESS_KEY not found");
         return NO;
     }
-    *accessKeyID = [[NSString alloc] initWithUTF8String:cAccessKey];
+    if (accessKey != nil) {
-    return YES;
+        *accessKey = account;
-    char *cSecretKey = getenv("ARQ_SECRET_KEY");
+    }
-    if (cSecretKey == NULL) {
+    if (secret != nil) {
-        SETNSERROR([AppKeychain errorDomain], ERROR_NOT_FOUND, @"ARQ_SECRET_KEY not found");
+        *secret = password;
-        return NO;
     }
-    *secret = [[NSString alloc] initWithUTF8String:cSecretKey];
     return YES;
 }
-+ (BOOL)encryptionKey:(NSString **)encryptionKey error:(NSError **)error {
++ (BOOL)containsEncryptionPasswordForS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID {
-    char *cEncryptionPassword = getenv("ARQ_ENCRYPTION_PASSWORD");
+    NSString *encryptionPassword = nil;
-    if (cEncryptionPassword != NULL) {
+    NSError *myError = nil;
-        SETNSERROR([AppKeychain errorDomain], ERROR_NOT_FOUND, @"ARQ_ENCRYPTION_PASSWORD not found");
+    return [AppKeychain encryptionPassword:&encryptionPassword forS3BucketName:theS3BucketName computerUUID:theComputerUUID error:&myError];
+}
++ (BOOL)encryptionPassword:(NSString **)encryptionPassword forS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID error:(NSError **)error {
+    NSString *label = [AppKeychain encryptionPasswordLabelForS3BucketName:theS3BucketName computerUUID:theComputerUUID];
+    NSString *account = nil;
+    NSString *password = nil;
+    if (![AppKeychain account:&account password:&password forLabel:label error:error]) {
         return NO;
     }
-    *encryptionKey = [[NSString alloc] initWithUTF8String:cEncryptionPassword];
+    if (encryptionPassword != nil) {
+        *encryptionPassword = password;
+    }
+    return YES;
+}
++ (BOOL)twitterAccessKey:(NSString **)theAccessKey secret:(NSString **)secret error:(NSError **)error {
+    return [AppKeychain account:theAccessKey password:secret forLabel:ARQ_TWITTER_LABEL error:error];
+}
+
+
+- (id)initWithBackupAppPath:(NSString *)theBackupAppPath 
+               agentAppPath:(NSString *)theAgentAppPath {
+    if (self = [super init]) {
+        backupAppPath = [theBackupAppPath copy];
+        agentAppPath = [theAgentAppPath copy];
+    }
+    return self;
+}
+- (void)dealloc {
+    if (access != NULL) {
+        CFRelease(access);
+    }
+    [backupAppPath release];
+    [agentAppPath release];
+    [super dealloc];
+}
+- (BOOL)setAccessKeyID:(NSString *)theAccessKeyID
+       secretAccessKey:(NSString *)theSecretAccessKey
+                 error:(NSError **)error {
+    return [self addLabel:ARQ_S3_LABEL account:theAccessKeyID password:theSecretAccessKey error:error];
+}
+- (BOOL)setEncryptionKey:(NSString *)theEncryptionPassword
+         forS3BucketName:(NSString *)theS3BucketName
+            computerUUID:(NSString *)theComputerUUID
+                   error:(NSError **)error {
+    NSString *label = [AppKeychain encryptionPasswordLabelForS3BucketName:theS3BucketName computerUUID:theComputerUUID];
+    return [self addLabel:label account:ARQ_ENCRYPTION_ACCOUNT_LABEL password:theEncryptionPassword error:error];
+}
+- (BOOL)setTwitterAccessKey:(NSString *)theKey secret:(NSString *)theSecret error:(NSError **)error {
+    return [self addLabel:ARQ_TWITTER_LABEL account:theKey password:theSecret error:error];
+}
+- (BOOL)deleteTwitterAccessKey:(NSError **)error {
+    return [self deleteLabel:ARQ_TWITTER_LABEL error:error];
+}
+@end
+
+@implementation AppKeychain (internal)
++ (NSString *)encryptionPasswordLabelForS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID {
+    return [NSString stringWithFormat:@"Arq Encryption:%@:%@", theS3BucketName, theComputerUUID];
+}
++ (BOOL)account:(NSString **)account password:(NSString **)password forLabel:(NSString *)theLabel error:(NSError **)error {
+    SecKeychainItemRef item = NULL;
+    if (![AppKeychain findItem:theLabel item:&item error:error]) {
+        return NO;
+    }
+    if (item == NULL) {
+        SETNSERROR(@"AppKeychainErrorDomain", -1, @"Keychain item %@ not found", theLabel);
+        return NO;
+    }
+    SecKeychainAttributeList *outAttrList = NULL;
+    UInt32 length;
+    void *data;
+    UInt32 tags[] = {
+        kSecAccountItemAttr
+    };
+    UInt32 formats[] = {
+        CSSM_DB_ATTRIBUTE_FORMAT_STRING
+    };
+    SecKeychainAttributeInfo info = {
+        1,
+        tags,
+        formats
+    };
+    OSStatus oss = SecKeychainItemCopyAttributesAndData(item, &info, NULL, &outAttrList, &length, &data);
+    if (oss != noErr) {
+        NSString *errorMessage = (NSString *)SecCopyErrorMessageString(oss, NULL);
+        SETNSERROR(@"AppKeychainErrorDomain", -1, @"Error reading data from Keychain item %@: %@ (code %d)", theLabel, errorMessage, oss);
+        [errorMessage release];
+        return NO;
+    }
+    if (account != nil) {
+        *account = nil;
+        for (UInt32 index = 0; index < outAttrList->count; index++) {
+            SecKeychainAttribute *attr = outAttrList->attr + index;
+            if (attr->tag == kSecAccountItemAttr) {
+                *account = [[[NSString alloc] initWithBytes:attr->data length:attr->length encoding:NSUTF8StringEncoding] autorelease];
+                break;
+            }
+        }
+    }
+    if (password != nil) {
+        *password = [[[NSString alloc] initWithBytes:data length:length encoding:NSUTF8StringEncoding] autorelease];
+    }
+    SecKeychainItemFreeAttributesAndData(outAttrList, data);
+    CFRelease(item);
+    return YES;
+}
++ (BOOL)findItem:(NSString *)theLabel item:(SecKeychainItemRef *)item error:(NSError **)error {
+    *item = NULL;
+    const char *label = [theLabel UTF8String];
+    SecKeychainAttribute attrs[] = {
+        {
+            kSecLabelItemAttr, 
+            strlen(label), 
+            (char *)label
+        }
+    };
+    SecKeychainAttributeList attributes = {
+        sizeof(attrs) / sizeof(attrs[0]),
+        attrs
+    };
+    SecKeychainSearchRef searchRef;
+    OSStatus oss;
+    oss = SecKeychainSearchCreateFromAttributes(NULL, kSecGenericPasswordItemClass, &attributes, &searchRef);
+    if (oss != noErr) {
+        SETNSERROR(@"AppKeychainErrorDomain", -1, @"error creating keychain search");
+        return NO;
+    }
+    SecKeychainSearchCopyNext(searchRef, item);
+    CFRelease(searchRef);
     return YES;
 }
 
+- (BOOL)loadSecAccess:(NSError **)error {
+    if (access == NULL) {
+        NSMutableArray *trustedApplications = [NSMutableArray array];
+        OSStatus oss;
+        SecTrustedApplicationRef agentApp;
+        oss = SecTrustedApplicationCreateFromPath([agentAppPath fileSystemRepresentation], &agentApp);
+        if (oss != noErr) {
+            CFStringRef msg = SecCopyErrorMessageString(oss, NULL);
+            SETNSERROR(@"AppKeychainErrorDomain", -1, @"Error creating Agent trusted application %@: %@ (code %d)", agentAppPath, (NSString *)msg, oss);
+            CFRelease(msg);
+            return NO;
+        }
+        [trustedApplications addObject:(id)agentApp];
+        CFRelease(agentApp);
+        
+        SecTrustedApplicationRef backupApp;
+        oss = SecTrustedApplicationCreateFromPath([backupAppPath fileSystemRepresentation], &backupApp);
+        if (oss != noErr) {
+            CFStringRef msg = SecCopyErrorMessageString(oss, NULL);
+            SETNSERROR(@"AppKeychainErrorDomain", -1, @"Error creating trusted application: %@ (code %d)", (NSString *)msg, oss);
+            CFRelease(agentApp);
+            CFRelease(msg);
+            return NO;
+        }
+        [trustedApplications addObject:(id)backupApp];
+        CFRelease(backupApp);
+    
+        
+        oss = SecAccessCreate((CFStringRef)@"Arq", (CFArrayRef)trustedApplications, &access);
+        if (oss != noErr) {
+            CFStringRef msg = SecCopyErrorMessageString(oss, NULL);
+            SETNSERROR(@"AppKeychainErrorDomain", -1, @"Error creating SecAccessRef: %@ (code %d)", (NSString *)msg, oss);
+            CFRelease(msg);
+            return NO;
+        }
+    }
+    return YES;
+}
+- (BOOL)deleteLabel:(NSString *)theLabel error:(NSError **)error {
+    SecKeychainItemRef item = NULL;
+    if (![AppKeychain findItem:theLabel item:&item error:error]) {
+        return NO;
+    }
+    if (item != NULL) {
+        OSStatus oss = SecKeychainItemDelete(item);
+        CFRelease(item);
+        if (oss != noErr) {
+            NSString *errorMessage = (NSString *)SecCopyErrorMessageString(oss, NULL);
+            SETNSERROR(@"AppKeychainErrorDomain", -1, @"error deleting item for label %@: %@ (code %d)", theLabel, errorMessage, oss);
+            [errorMessage release];
+            return NO;
+        }
+    }
+    return YES;
+}
+- (BOOL)addLabel:(NSString *)theLabel account:(NSString *)theAccount password:(NSString *)thePassword error:(NSError **)error {
+    if (![self loadSecAccess:error]) {
+        return NO;
+    }
+    if (![self deleteLabel:theLabel error:error]) {
+        return NO;
+    }
+    const char *label = [theLabel UTF8String];
+    const char *account = [theAccount UTF8String];
+    const char *password = [thePassword UTF8String];
+    SecKeychainAttribute attrs[] = {
+        {
+            kSecLabelItemAttr, 
+            strlen(label), 
+            (char *)label
+        },
+        {
+            kSecAccountItemAttr,
+            strlen(account),
+            (char *)account
+        },
+        {
+            kSecServiceItemAttr,
+            strlen(label),
+            (char *)label
+        }
+    };
+    SecKeychainAttributeList attributes = {
+        sizeof(attrs) / sizeof(attrs[0]),
+        attrs
+    };
+    OSStatus oss = SecKeychainItemCreateFromContent(kSecGenericPasswordItemClass, &attributes, strlen(password), password, NULL, access, NULL);
+    if (oss != noErr) {
+        NSString *errorMessage = (NSString *)SecCopyErrorMessageString(oss, NULL);
+        SETNSERROR(@"AppKeychainErrorDomain", -1, @"Error creating keychain item: %@ (code %d)", errorMessage, oss);
+        [errorMessage release];
+        return NO;
+    }
+    return YES;
+}
 @end

ArqFark.h

@@ -20,7 +20,7 @@
 - (id)initWithS3Service:(S3Service *)theS3
            s3BucketName:(NSString *)theS3BucketName
            computerUUID:(NSString *)theComputerUUID;
-- (NSData *)bucketDataForPath:(NSString *)bucketDataPath error:(NSError **)error;
+- (NSData *)bucketDataForRelativePath:(NSString *)bucketDataRelativePath error:(NSError **)error;
 - (NSData *)dataForSHA1:(NSString *)sha1 error:(NSError **)error;
 - (ServerBlob *)newServerBlobForSHA1:(NSString *)sha1 error:(NSError **)error;
 @end

ArqFark.m

@@ -41,14 +41,14 @@
     [creatorThread release];
     [super dealloc];
 }
-- (NSData *)bucketDataForPath:(NSString *)bucketDataPath error:(NSError **)error {
+- (NSData *)bucketDataForRelativePath:(NSString *)bucketDataRelativePath error:(NSError **)error {
     NSAssert([NSThread currentThread] == creatorThread, @"must be on same thread!");
     
     NSError *myError = nil;
-    NSData *data = [s3 dataAtPath:[FarkPath s3PathForBucketDataPath:bucketDataPath s3BucketName:s3BucketName computerUUID:computerUUID] error:&myError];
+    NSData *data = [s3 dataAtPath:[FarkPath s3PathForBucketDataRelativePath:bucketDataRelativePath s3BucketName:s3BucketName computerUUID:computerUUID] error:&myError];
     if (data == nil) {
         if ([myError isErrorWithDomain:[S3Service errorDomain] code:ERROR_NOT_FOUND]) {
-            SETNSERROR([ArqFark errorDomain], ERROR_NOT_FOUND, @"bucket data not found for path %@", bucketDataPath);
+            SETNSERROR([ArqFark errorDomain], ERROR_NOT_FOUND, @"bucket data not found for path %@", bucketDataRelativePath);
         } else {
             if (error != NULL) {
                 *error = myError;
@@ -67,7 +67,7 @@
     return data;
 }
 - (ServerBlob *)newServerBlobForSHA1:(NSString *)sha1 error:(NSError **)error {
-    NSAssert([NSThread currentThread] == creatorThread, @"must be on same thread!");
+//    NSAssert([NSThread currentThread] == creatorThread, @"must be on same thread!");
     NSString *s3Path = [NSString stringWithFormat:@"/%@/%@/objects/%@", s3BucketName, computerUUID, sha1];
     return [s3 newServerBlobAtPath:s3Path error:error];
 }

ArqFolder.h

@@ -1,43 +0,0 @@
-/*
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
-
-#import <Cocoa/Cocoa.h>
-#import "DictNode.h"
-
-@interface ArqFolder : NSObject {
-    NSString *s3Path;
-    NSString *localPath;
-}
-- (id)initWithS3Path:(NSString *)theS3Path plist:(DictNode *)plist;
-- (NSString *)s3Path;
-- (NSString *)localPath;
-@end

ArqPackSet.h

@@ -18,12 +18,11 @@
     NSDictionary *packIndexEntries;
 }
 + (NSString *)errorDomain;
-+ (BOOL)cachePackSetIndexesForS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID error:(NSError **)error;
 - (id)initWithS3Service:(S3Service *)theS3
            s3BucketName:(NSString *)theS3BucketName
            computerUUID:(NSString *)theComputerUUID
             packSetName:(NSString *)thePackSetName;
 - (NSString *)packSetName;
 - (ServerBlob *)newServerBlobForSHA1:(NSString *)sha1 error:(NSError **)error;
-- (BOOL)packSHA1:(NSString **)packSHA1 forPackedSHA1:(NSString *)packedSHA1 error:(NSError **)error;
+- (BOOL)containsBlob:(BOOL *)contains forSHA1:(NSString *)sha1 packSHA1:(NSString **)packSHA1 error:(NSError **)error;
 @end

ArqPackSet.m

@@ -22,6 +22,7 @@
 
 @interface ArqPackSet (internal)
 - (ServerBlob *)newInternalServerBlobForSHA1:(NSString *)sha1 error:(NSError **)error;
+- (BOOL)loadPackIndexEntries:(NSError **)error;
 - (NSDictionary *)doLoadPackIndexEntries:(NSError **)error;
 @end
 
@@ -29,33 +30,6 @@
 + (NSString *)errorDomain {
     return @"ArqPackSetErrorDomain";
 }
-+ (BOOL)cachePackSetIndexesForS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID error:(NSError **)error {
-    NSString *accessKeyID;
-    NSString *secretAccessKey;
-    if (![AppKeychain accessKeyID:&accessKeyID secretAccessKey:&secretAccessKey error:error]) {
-        return NO;
-    }
-    S3AuthorizationProvider *sap = [[[S3AuthorizationProvider alloc] initWithAccessKey:accessKeyID secretKey:secretAccessKey] autorelease];
-    S3Service *theS3 = [[[S3Service alloc] initWithS3AuthorizationProvider:sap useSSL:YES retryOnNetworkError:YES] autorelease];
-    NSString *packSetsPrefix = [NSString stringWithFormat:@"/%@/%@/packsets/", theS3BucketName, theComputerUUID];
-    NSArray *packPaths = [theS3 pathsWithPrefix:packSetsPrefix error:error];
-    if (packPaths == nil) {
-        return NO;
-    }
-    for (NSString *packPath in packPaths) {
-        NSString *pattern = [NSString stringWithFormat:@"/%@/%@/packsets/([^/]+)/(\\w+)\\.pack",  theS3BucketName, theComputerUUID];
-        NSRange sha1Range = [packPath rangeOfRegex:pattern capture:2];
-        if (sha1Range.location != NSNotFound) {
-            NSString *packSHA1 = [packPath substringWithRange:sha1Range];
-            NSString *thePackSetName = [packPath substringWithRange:[packPath rangeOfRegex:pattern capture:1]];
-            DiskPackIndex *dpi = [[[DiskPackIndex alloc] initWithS3Service:theS3 s3BucketName:theS3BucketName computerUUID:theComputerUUID packSetName:thePackSetName packSHA1:packSHA1] autorelease];
-            if (![dpi makeLocal:error]) {
-                return NO;
-            }
-        }
-    }
-    return YES;
-}
 
 - (id)initWithS3Service:(S3Service *)theS3
            s3BucketName:(NSString *)theS3BucketName
@@ -80,26 +54,15 @@
 - (NSString *)packSetName {
     return packSetName;
 }
-- (BOOL)packSHA1:(NSString **)packSHA1 forPackedSHA1:(NSString *)packedSHA1 error:(NSError **)error {
-    if (packIndexEntries == nil) {
-        NSDictionary *entries = [self doLoadPackIndexEntries:error];
-        if (entries == nil) {
-            return NO;
-        }
-        packIndexEntries = [entries retain];
-    }
-    *packSHA1 = nil;
-    PackIndexEntry *pie = [packIndexEntries objectForKey:packedSHA1];
-    if (pie != nil) {
-        *packSHA1 = [pie packSHA1];
-    }
-    return YES;
-}
 - (ServerBlob *)newServerBlobForSHA1:(NSString *)sha1 error:(NSError **)error {
     ServerBlob *sb = nil;
     NSError *myError = nil;
     NSUInteger i = 0;
+    NSAutoreleasePool *pool = nil;
     for (i = 0; i < MAX_RETRIES; i++) {
+        [pool drain];
+        pool = [[NSAutoreleasePool alloc] init];
+        myError = nil;
         sb = [self newInternalServerBlobForSHA1:sha1 error:&myError];
         if (sb == nil) {
             if ([myError isErrorWithDomain:[ArqPackSet errorDomain] code:ERROR_PACK_INDEX_ENTRY_NOT_RESOLVABLE]) {
@@ -113,6 +76,9 @@
             break;
         }
     }
+    [myError retain];
+    [pool drain];
+    [myError autorelease];
     if (sb == nil) {
         if ([myError isErrorWithDomain:[ArqPackSet errorDomain] code:ERROR_PACK_INDEX_ENTRY_NOT_RESOLVABLE]) {
             SETNSERROR([ArqPackSet errorDomain], ERROR_NOT_FOUND, @"failed %u times to load blob for sha1 %@ from pack set %@", i, sha1, packSetName);
@@ -122,23 +88,36 @@
     }
     return sb;
 }
+- (BOOL)containsBlob:(BOOL *)contains forSHA1:(NSString *)sha1 packSHA1:(NSString **)packSHA1 error:(NSError **)error {
+    if (packIndexEntries == nil && ![self loadPackIndexEntries:error]) {
+        return NO;
+    }
+    PackIndexEntry *pie = [packIndexEntries objectForKey:sha1];
+    *contains = (pie != nil);
+    if (pie != nil) {
+        *packSHA1 = [pie packSHA1];
+    }
+    return YES;
+}
 @end
 
 @implementation ArqPackSet (internal)
 - (ServerBlob *)newInternalServerBlobForSHA1:(NSString *)sha1 error:(NSError **)error {
-    if (packIndexEntries == nil) {
+    if (packIndexEntries == nil && ![self loadPackIndexEntries:error]) {
-        NSDictionary *entries = [self doLoadPackIndexEntries:error];
+        return nil;
-        if (entries == nil) {
-            return nil;
-        }
-        packIndexEntries = [entries retain];
     }
     PackIndexEntry *pie = [packIndexEntries objectForKey:sha1];
     if (pie == nil) {
         SETNSERROR([ArqPackSet errorDomain], ERROR_NOT_FOUND, @"sha1 %@ not found in pack set %@", sha1, packSetName);
         return NO;
     }
-    DiskPack *diskPack = [[DiskPack alloc] initWithS3Service:s3 s3BucketName:s3BucketName computerUUID:computerUUID packSetName:packSetName packSHA1:[pie packSHA1]];
+    DiskPack *diskPack = [[DiskPack alloc] initWithS3Service:s3 
+                                                s3BucketName:s3BucketName 
+                                                computerUUID:computerUUID 
+                                                 packSetName:packSetName 
+                                                    packSHA1:[pie packSHA1]
+                                                   targetUID:getuid()
+                                                   targetGID:getgid()];
     ServerBlob *sb = nil;
     do {
         NSError *myError = nil;
@@ -157,6 +136,38 @@
     [diskPack release];
     return sb;
 }
+- (BOOL)loadPackIndexEntries:(NSError **)error {
+    BOOL ret = YES;
+    NSAutoreleasePool *pool = nil;
+    for (;;) {
+        [pool drain];
+        pool = [[NSAutoreleasePool alloc] init];
+        NSError *myError = nil;
+        NSDictionary *entries = [self doLoadPackIndexEntries:&myError];
+        if (entries != nil) {
+            packIndexEntries = [entries retain];
+            break;
+        }
+        if ([myError code] != ERROR_NOT_FOUND) {
+            if (error != NULL) {
+                *error = myError;
+            }
+            ret = NO;
+            break;
+        }
+        // If it's a not-found error, it can be because Arq Agent replaced a pack with another one between when we got
+        // the S3 list and when we tried to make them local.
+        HSLogDebug(@"error loading pack index entries (retrying): %@", myError);
+    }
+    if (!ret && error != NULL) {
+        [*error retain];
+    }
+    [pool drain];
+    if (!ret && error != NULL) {
+        [*error autorelease];
+    }
+    return ret;
+}
 - (NSDictionary *)doLoadPackIndexEntries:(NSError **)error {
     NSMutableDictionary *entries = [NSMutableDictionary dictionary];
     NSString *packSHA1Prefix = [NSString stringWithFormat:@"/%@/%@/packsets/%@/", s3BucketName, computerUUID, packSetName];
@@ -169,7 +180,13 @@
         if (sha1Range.location != NSNotFound) {
             NSString *packSHA1 = [packSHA1Path substringWithRange:sha1Range];
             BOOL ret = NO;
-            DiskPackIndex *index = [[DiskPackIndex alloc] initWithS3Service:s3 s3BucketName:s3BucketName computerUUID:computerUUID packSetName:packSetName packSHA1:packSHA1];
+            DiskPackIndex *index = [[DiskPackIndex alloc] initWithS3Service:s3 
+                                                               s3BucketName:s3BucketName 
+                                                               computerUUID:computerUUID 
+                                                                packSetName:packSetName 
+                                                                   packSHA1:packSHA1
+                                                                  targetUID:getuid()
+                                                                  targetGID:getgid()];
             do {
                 if (![index makeLocal:error]) {
                     break;
@@ -178,7 +195,7 @@
                 if (pies == nil) {
                     break;
                 }
-                HSLogDebug(@"found %u entries in s3 pack sha1 %@ packset %@ computer %@ s3bucket %@", [pies count], packSHA1, packSetName, computerUUID, s3BucketName);
+                HSLogTrace(@"found %u entries in s3 pack sha1 %@ packset %@ computer %@ s3bucket %@", [pies count], packSHA1, packSetName, computerUUID, s3BucketName);
                 for (PackIndexEntry *pie in pies) {
                     [entries setObject:pie forKey:[pie objectSHA1]];
                 }

ArqRepo.h

@@ -13,11 +13,14 @@
 @class Commit;
 @class Tree;
 @class ServerBlob;
+@class CryptoKey;
+@class BlobKey;
 
 @interface ArqRepo : NSObject {
     NSString *bucketUUID;
-    NSString *encryptionKey;
     ArqFark *arqFark;
+    CryptoKey *cryptoKey;
+    CryptoKey *stretchedCryptoKey;
     ArqPackSet *treesPackSet;
     ArqPackSet *blobsPackSet;
 }
@@ -26,12 +29,15 @@
            s3BucketName:(NSString *)theS3BucketName 
            computerUUID:(NSString *)theComputerUUID
              bucketUUID:(NSString *)theBucketUUID 
-          encryptionKey:(NSString *)theEncryptionKey;
+     encryptionPassword:(NSString *)theEncryptionPassword
+                   salt:(NSData *)theEncryptionSalt
+                  error:(NSError **)error;
 
-- (NSString *)headSHA1:(NSError **)error;
+- (NSString *)bucketUUID;
-- (Commit *)commitForSHA1:(NSString *)theSHA1 error:(NSError **)error;
+- (BlobKey *)headBlobKey:(NSError **)error;
-- (Tree *)treeForSHA1:(NSString *)theSHA1 error:(NSError **)error;
+- (Commit *)commitForBlobKey:(BlobKey *)treeBlobKey error:(NSError **)error;
-- (NSData *)blobDataForSHA1:(NSString *)sha1 error:(NSError **)error;
+- (Tree *)treeForBlobKey:(BlobKey *)treeBlobKey error:(NSError **)error;
-- (NSData *)blobDataForSHA1s:(NSArray *)sha1s error:(NSError **)error;
+- (NSData *)blobDataForBlobKey:(BlobKey *)treeBlobKey error:(NSError **)error;
-- (ServerBlob *)newServerBlobForSHA1:(NSString *)sha1 error:(NSError **)error;
+- (ServerBlob *)newServerBlobForBlobKey:(BlobKey *)treeBlobKey error:(NSError **)error;
+- (BOOL)containsPackedBlob:(BOOL *)contains forBlobKey:(BlobKey *)theBlobKey packSetName:(NSString **)packSetName packSHA1:(NSString **)packSHA1 error:(NSError **)error;
 @end

ArqRepo.m

@@ -18,6 +18,10 @@
 #import "NSData-Encrypt.h"
 #import "SetNSError.h"
 #import "NSError_extra.h"
+#import "GunzipInputStream.h"
+#import "CryptoKey.h"
+#import "BlobKey.h"
+#import "Encryption.h"
 
 @implementation ArqRepo
 + (NSString *)errorDomain {
@@ -27,10 +31,29 @@
            s3BucketName:(NSString *)theS3BucketName 
            computerUUID:(NSString *)theComputerUUID
              bucketUUID:(NSString *)theBucketUUID 
-          encryptionKey:(NSString *)theEncryptionKey {
+     encryptionPassword:(NSString *)theEncryptionPassword 
+                   salt:(NSData *)theEncryptionSalt
+                  error:(NSError **)error {
     if (self = [super init]) {
         bucketUUID = [theBucketUUID retain];
-        encryptionKey = [theEncryptionKey retain];
+        
+        if (theEncryptionPassword == nil) {
+            SETNSERROR([Encryption errorDomain], -1, @"missing encryption password");
+            [self release];
+            return nil;
+        }
+        
+        cryptoKey = [[CryptoKey alloc] initLegacyWithPassword:theEncryptionPassword error:error];
+        if (cryptoKey == nil) {
+            [self release];
+            return nil;
+        }
+        stretchedCryptoKey = [[CryptoKey alloc] initWithPassword:theEncryptionPassword salt:theEncryptionSalt error:error];
+        if (stretchedCryptoKey == nil) {
+            [self release];
+            return nil;
+        }
+
         arqFark = [[ArqFark alloc] initWithS3Service:theS3 s3BucketName:theS3BucketName computerUUID:theComputerUUID];
         treesPackSet = [[ArqPackSet alloc] initWithS3Service:theS3 s3BucketName:theS3BucketName computerUUID:theComputerUUID packSetName:[theBucketUUID stringByAppendingString:@"-trees"]];
         blobsPackSet = [[ArqPackSet alloc] initWithS3Service:theS3 s3BucketName:theS3BucketName computerUUID:theComputerUUID packSetName:[theBucketUUID stringByAppendingString:@"-blobs"]];
@@ -39,16 +62,20 @@
 }
 - (void)dealloc {
     [bucketUUID release];
-    [encryptionKey release];
+    [cryptoKey release];
+    [stretchedCryptoKey release];
     [arqFark release];
     [treesPackSet release];
     [blobsPackSet release];
     [super dealloc];
 }
-- (NSString *)headSHA1:(NSError **)error {
+- (NSString *)bucketUUID {
-    NSString *bucketDataPath = [NSString stringWithFormat:@"/%@/refs/heads/master", bucketUUID];
+    return bucketUUID;
+}
+- (BlobKey *)headBlobKey:(NSError **)error {
+    NSString *bucketDataRelativePath = [NSString stringWithFormat:@"/%@/refs/heads/master", bucketUUID];
     NSError *myError = nil;
-    NSData *data = [arqFark bucketDataForPath:bucketDataPath error:&myError];
+    NSData *data = [arqFark bucketDataForRelativePath:bucketDataRelativePath error:&myError];
     if (data == nil) {
         if ([myError isErrorWithDomain:[ArqFark errorDomain] code:ERROR_NOT_FOUND]) {
             SETNSERROR([ArqRepo errorDomain], ERROR_NOT_FOUND, @"no head for bucketUUID %@", bucketUUID);
@@ -59,28 +86,39 @@
         }
         return nil;
     }
-    return [[[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] autorelease];
+    NSString *sha1 = [[[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] autorelease];
+    BOOL stretch = NO;
+    if ([sha1 length] > 40) {
+        stretch = [sha1 characterAtIndex:40] == 'Y';
+        sha1 = [sha1 substringToIndex:40];
+    }
+    return [[[BlobKey alloc] initWithSHA1:sha1 stretchEncryptionKey:stretch] autorelease];
 }
-- (Commit *)commitForSHA1:(NSString *)theSHA1 error:(NSError **)error {
+- (Commit *)commitForBlobKey:(BlobKey *)commitBlobKey error:(NSError **)error {
     NSError *myError = nil;
-    ServerBlob *sb = [treesPackSet newServerBlobForSHA1:theSHA1 error:&myError];
+    ServerBlob *sb = [treesPackSet newServerBlobForSHA1:[commitBlobKey sha1] error:&myError];
     if (sb == nil) {
         if ([myError isErrorWithDomain:[ArqPackSet errorDomain] code:ERROR_NOT_FOUND]) {
-            HSLogDebug(@"commit %@ not found in pack set %@", theSHA1, [treesPackSet packSetName]);
+            HSLogDebug(@"commit %@ not found in pack set %@", commitBlobKey, [treesPackSet packSetName]);
-            SETNSERROR([ArqRepo errorDomain], ERROR_NOT_FOUND, @"commit not found for sha1 %@", theSHA1);
+            SETNSERROR([ArqRepo errorDomain], ERROR_NOT_FOUND, @"commit %@ not found", commitBlobKey);
         } else {
-            HSLogError(@"commit not found for %@: %@", theSHA1, [myError localizedDescription]);
+            HSLogError(@"commit %@ not found for: %@", commitBlobKey, [myError localizedDescription]);
             if (error != NULL) {
                 *error = myError;
             }
         }
         return nil;
     }
-    NSData *data = [[sb slurp:error] decryptWithCipher:ARQ_DEFAULT_CIPHER_NAME key:encryptionKey error:error];
+    NSData *encrypted = [sb slurp:error];
     [sb release];
+    if (encrypted == nil) {
+        return nil;
+    }
+    NSData *data = [encrypted decryptWithCryptoKey:([commitBlobKey stretchEncryptionKey] ? stretchedCryptoKey : cryptoKey) error:error];
     if (data == nil) {
         return nil;
     }
+
     DataInputStream *dis = [[DataInputStream alloc] initWithData:data];
     BufferedInputStream *bis = [[BufferedInputStream alloc] initWithUnderlyingStream:dis];
     Commit *commit = [[[Commit alloc] initWithBufferedInputStream:bis error:error] autorelease];
@@ -90,26 +128,31 @@
 }
 
 // Returns NO if commit not found:
-- (Tree *)treeForSHA1:(NSString *)theSHA1 error:(NSError **)error {
+- (Tree *)treeForBlobKey:(BlobKey *)blobKey error:(NSError **)error {
     NSError *myError = nil;
-    ServerBlob *sb = [treesPackSet newServerBlobForSHA1:theSHA1 error:error];
+    ServerBlob *sb = [treesPackSet newServerBlobForSHA1:[blobKey sha1] error:&myError];
     if (sb == nil) {
         if ([myError isErrorWithDomain:[ArqPackSet errorDomain] code:ERROR_NOT_FOUND]) {
-            HSLogDebug(@"tree %@ not found in pack set %@", theSHA1, [treesPackSet packSetName]);
+            HSLogDebug(@"tree %@ not found in pack set %@", blobKey, [treesPackSet packSetName]);
-            SETNSERROR([ArqRepo errorDomain], ERROR_NOT_FOUND, @"commit not found for sha1 %@", theSHA1);
+            SETNSERROR([ArqRepo errorDomain], ERROR_NOT_FOUND, @"tree %@ not found", blobKey);
         } else {
-            HSLogError(@"tree not found for %@: %@", theSHA1, [myError localizedDescription]);
+            HSLogError(@"error reading tree %@: %@", blobKey, [myError localizedDescription]);
             if (error != NULL) {
                 *error = myError;
             }
         }
         return nil;
     }
-    NSData *data = [[sb slurp:error] decryptWithCipher:ARQ_DEFAULT_CIPHER_NAME key:encryptionKey error:error];
+    NSData *encrypted = [sb slurp:error];
     [sb release];
+    if (encrypted == nil) {
+        return nil;
+    }
+    NSData *data = [encrypted decryptWithCryptoKey:([blobKey stretchEncryptionKey] ? stretchedCryptoKey : cryptoKey) error:error];
     if (data == nil) {
         return nil;
     }
+
     DataInputStream *dis = [[DataInputStream alloc] initWithData:data];
     BufferedInputStream *bis = [[BufferedInputStream alloc] initWithUnderlyingStream:dis];
     Tree *tree = [[[Tree alloc] initWithBufferedInputStream:bis error:error] autorelease];
@@ -117,8 +160,8 @@
     [dis release];
     return tree;
 }
-- (NSData *)blobDataForSHA1:(NSString *)sha1 error:(NSError **)error {
+- (NSData *)blobDataForBlobKey:(BlobKey *)treeBlobKey error:(NSError **)error {
-    ServerBlob *sb = [self newServerBlobForSHA1:sha1 error:error];
+    ServerBlob *sb = [self newServerBlobForBlobKey:treeBlobKey error:error];
     if (sb == nil) {
         return nil;
     }
@@ -126,28 +169,16 @@
     [sb release];
     return data;
 }
-- (NSData *)blobDataForSHA1s:(NSArray *)sha1s error:(NSError **)error {
+- (ServerBlob *)newServerBlobForBlobKey:(BlobKey *)theBlobKey error:(NSError **)error {
-    //FIXME: This is very inefficient!
-    NSMutableData *ret = [NSMutableData data];
-    for (NSString *sha1 in sha1s) {
-        NSData *data = [self blobDataForSHA1:sha1 error:error];
-        if (data == nil) {
-            return nil;
-        }
-        [ret appendData:data];
-    }
-    return ret;
-}
-- (ServerBlob *)newServerBlobForSHA1:(NSString *)sha1 error:(NSError **)error {
     NSError *myError = nil;
-    ServerBlob *sb = [blobsPackSet newServerBlobForSHA1:sha1 error:&myError];
+    ServerBlob *sb = [blobsPackSet newServerBlobForSHA1:[theBlobKey sha1] error:&myError];
     if (sb == nil) {
         if ([myError isErrorWithDomain:[ArqPackSet errorDomain] code:ERROR_NOT_FOUND]) {
-            HSLogTrace(@"sha1 %@ not found in pack set %@; looking in S3", sha1, [blobsPackSet packSetName]);
+            HSLogTrace(@"%@ not found in pack set %@; looking in S3", theBlobKey, [blobsPackSet packSetName]);
-            sb = [arqFark newServerBlobForSHA1:sha1 error:&myError];
+            sb = [arqFark newServerBlobForSHA1:[theBlobKey sha1] error:&myError];
             if (sb == nil) {
                 if ([myError isErrorWithDomain:[ArqFark errorDomain] code:ERROR_NOT_FOUND]) {
-                    SETNSERROR([ArqRepo errorDomain], ERROR_NOT_FOUND, @"sha1 %@ not found", sha1);
+                    SETNSERROR([ArqRepo errorDomain], ERROR_NOT_FOUND, @"object %@ not found", theBlobKey);
                 } else {
                     if (error != NULL) {
                         *error = myError;
@@ -161,21 +192,45 @@
             }
         }
     }
-    if (sb != nil) {
+    if (sb == nil) {
-        id <InputStream> is = [sb newInputStream];
+        return nil;
-        NSString *mimeType = [sb mimeType];
-        NSString *downloadName = [sb downloadName];
-        [sb autorelease];
-        sb = nil;
-        DecryptedInputStream *dis = [[DecryptedInputStream alloc] initWithInputStream:is cipherName:ARQ_DEFAULT_CIPHER_NAME key:encryptionKey error:error];
-        [is release];
-        if (dis != nil) {
-            sb = [[ServerBlob alloc] initWithInputStream:dis mimeType:mimeType downloadName:downloadName];
-            [dis release];
-        }
     }
+
+    NSString *mimeType = [sb mimeType];
+    NSString *downloadName = [sb downloadName];
+    NSData *encrypted = [sb slurp:error];
+    [sb release];
+    sb = nil;
+    if (encrypted == nil) {
+        return nil;
+    }
+    NSData *data = [encrypted decryptWithCryptoKey:([theBlobKey stretchEncryptionKey] ? stretchedCryptoKey : cryptoKey) error:error];
+    if (data == nil) {
+        return nil;
+    }
+    id <InputStream> blobIS = [[DataInputStream alloc] initWithData:data];
+    sb = [[ServerBlob alloc] initWithInputStream:blobIS mimeType:mimeType downloadName:downloadName];
+    [blobIS release];
     return sb;
 }
+- (BOOL)containsPackedBlob:(BOOL *)contains forBlobKey:(BlobKey *)theBlobKey packSetName:(NSString **)packSetName packSHA1:(NSString **)packSHA1 error:(NSError **)error {
+    if (![blobsPackSet containsBlob:contains forSHA1:[theBlobKey sha1] packSHA1:packSHA1 error:error]) {
+        return NO;
+    }
+    if (*contains) {
+        *packSetName = [blobsPackSet packSetName];
+        return YES;
+    }
+    
+    if (![treesPackSet containsBlob:contains forSHA1:[theBlobKey sha1] packSHA1:packSHA1 error:error]) {
+        return NO;
+    }
+    if (*contains) {
+        *packSetName = [treesPackSet packSetName];
+    }
+    return YES;
+}
+
 
 #pragma mark NSObject
 - (NSString *)description {

ArqRepo_Verifier.h

@@ -1,15 +0,0 @@
-//
-//  ArqRepo_Verifier.h
-//  arq_restore
-//
-//  Created by Stefan Reitshamer on 8/19/10.
-//  Copyright 2010 __MyCompanyName__. All rights reserved.
-//
-
-#import <Cocoa/Cocoa.h>
-#import "ArqRepo.h"
-
-@interface ArqRepo (Verifier)
-- (NSString *)blobsPackSetName;
-- (BOOL)packSHA1:(NSString **)packSHA1 forPackedBlobSHA1:(NSString *)sha1 error:(NSError **)error;
-@end

ArqRepo_Verifier.m

@@ -1,19 +0,0 @@
-//
-//  ArqRepo_Verifier.m
-//  arq_restore
-//
-//  Created by Stefan Reitshamer on 8/19/10.
-//  Copyright 2010 __MyCompanyName__. All rights reserved.
-//
-
-#import "ArqRepo_Verifier.h"
-#import "ArqPackSet.h"
-
-@implementation ArqRepo (Verifier)
-- (NSString *)blobsPackSetName {
-    return [blobsPackSet packSetName];
-}
-- (BOOL)packSHA1:(NSString **)packSHA1 forPackedBlobSHA1:(NSString *)sha1 error:(NSError **)error {
-    return [blobsPackSet packSHA1:packSHA1 forPackedSHA1:sha1 error:error];
-}
-@end

ArqRestoreCommand.m

@@ -36,12 +36,13 @@
 #import "S3Service.h"
 #import "RegexKitLite.h"
 #import "DictNode.h"
-#import "ArqFolder.h"
 #import "HTTP.h"
 #import "Restorer.h"
 #import "NSErrorCodes.h"
 #import "NSError_extra.h"
 #import "UserAndComputer.h"
+#import "ArqSalt.h"
+#import "ArqRepo.h"
 
 @interface ArqRestoreCommand (internal)
 - (BOOL)printArqFolders:(NSError **)error;
@@ -66,7 +67,7 @@
         }
         if (accessKey != nil && secretKey != nil) {
             S3AuthorizationProvider *sap = [[S3AuthorizationProvider alloc] initWithAccessKey:accessKey secretKey:secretKey];
-            s3 = [[S3Service alloc] initWithS3AuthorizationProvider:sap useSSL:NO retryOnNetworkError:YES];
+            s3 = [[S3Service alloc] initWithS3AuthorizationProvider:sap useSSL:NO retryOnTransientError:YES];
             [sap release];
         }
     }
@@ -213,7 +214,23 @@
     }
     printf(" to %s/%s\n", [[[NSFileManager defaultManager] currentDirectoryPath] UTF8String], [bucketName UTF8String]);
     
-    Restorer *restorer = [[[Restorer alloc] initWithS3Service:s3 s3BucketName:s3BucketName computerUUID:computerUUID bucketUUID:bucketUUID bucketName:bucketName encryptionKey:encryptionPassword] autorelease];
+    NSError *saltError = nil;
+    ArqSalt *arqSalt = [[[ArqSalt alloc] initWithAccessKeyID:accessKey secretAccessKey:secretKey s3BucketName:s3BucketName computerUUID:computerUUID] autorelease];
+    NSData *salt = [arqSalt salt:&saltError];
+    if (salt == nil) {
+        if ([saltError code] != ERROR_NOT_FOUND) {
+            if (error != NULL) {
+                *error = saltError;
+            }
+            return NO;
+        }
+    }
+    ArqRepo *repo = [[[ArqRepo alloc] initWithS3Service:s3 s3BucketName:s3BucketName computerUUID:computerUUID bucketUUID:bucketUUID encryptionPassword:encryptionPassword salt:salt error:error] autorelease];
+    if (repo == nil) {
+        return NO;
+    }
+    
+    Restorer *restorer = [[[Restorer alloc] initWithRepo:repo bucketName:bucketName] autorelease];
     if (![restorer restore:error]) {
         return NO;
     }

ArqSalt.h

@@ -0,0 +1,25 @@
+//
+//  ArqSalt.h
+//  Arq
+//
+//  Created by Stefan Reitshamer on 7/16/11.
+//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#import <Cocoa/Cocoa.h>
+
+
+@interface ArqSalt : NSObject {
+    NSString *accessKeyID;
+    NSString *secretAccessKey;
+    NSString *s3BucketName;
+    NSString *computerUUID;
+    NSString *localPath;
+    NSString *s3Path;
+}
+- (id)initWithAccessKeyID:(NSString *)theAccessKeyID
+          secretAccessKey:(NSString *)theSecretAccessKey
+             s3BucketName:(NSString *)theS3BucketName
+             computerUUID:(NSString *)theComputerUUID;
+- (NSData *)salt:(NSError **)error;
+@end

ArqSalt.m

@@ -0,0 +1,75 @@
+//
+//  ArqSalt.m
+//  Arq
+//
+//  Created by Stefan Reitshamer on 7/16/11.
+//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#import "ArqSalt.h"
+#import "S3AuthorizationProvider.h"
+#import "S3Service.h"
+#import "Blob.h"
+#import "BlobACL.h"
+#import "NSFileManager_extra.h"
+#import "UserLibrary_Arq.h"
+
+#define SALT_LENGTH (8)
+
+@interface ArqSalt (internal)
+- (NSData *)createRandomSalt;
+@end
+
+@implementation ArqSalt
+- (id)initWithAccessKeyID:(NSString *)theAccessKeyID
+          secretAccessKey:(NSString *)theSecretAccessKey
+             s3BucketName:(NSString *)theS3BucketName
+             computerUUID:(NSString *)theComputerUUID {
+    if (self = [super init]) {
+        accessKeyID = [theAccessKeyID retain];
+        secretAccessKey = [theSecretAccessKey retain];
+        s3BucketName = [theS3BucketName retain];
+        computerUUID = [theComputerUUID retain];
+        localPath = [[NSString alloc] initWithFormat:@"%@/Cache.noindex/%@/%@/salt.dat", [UserLibrary arqUserLibraryPath], s3BucketName, computerUUID];
+        s3Path = [[NSString alloc] initWithFormat:@"/%@/%@/salt", s3BucketName, computerUUID];
+    }
+    return self;
+}
+- (void)dealloc {
+    [accessKeyID release];
+    [secretAccessKey release];
+    [s3BucketName release];
+    [computerUUID release];
+    [localPath release];
+    [s3Path release];
+    [super dealloc];
+}
+
+- (NSData *)salt:(NSError **)error {
+    NSData *ret = [NSData dataWithContentsOfFile:localPath options:NSUncachedRead error:error];
+    if (ret == nil) {
+        S3AuthorizationProvider *sap = [[[S3AuthorizationProvider alloc] initWithAccessKey:accessKeyID secretKey:secretAccessKey] autorelease];
+        S3Service *s3 = [[[S3Service alloc] initWithS3AuthorizationProvider:sap useSSL:YES retryOnTransientError:NO] autorelease];
+        ret = [s3 dataAtPath:s3Path error:error];
+        if (ret == nil) {
+            return nil;
+        }
+        NSError *myError = nil;
+        if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:localPath error:&myError]
+            || ![ret writeToFile:localPath options:NSAtomicWrite error:&myError]) {
+            HSLogError(@"error caching salt data to %@: %@", localPath, myError);
+        }
+    }
+    return ret;
+}
+@end
+
+@implementation ArqSalt (internal)
+- (NSData *)createRandomSalt {
+    unsigned char buf[SALT_LENGTH];
+    for (NSUInteger i = 0; i < SALT_LENGTH; i++) {
+        buf[i] = (unsigned char)(rand() % 256);
+    }
+    return [[[NSData alloc] initWithBytes:buf length:SALT_LENGTH] autorelease];
+}
+@end

ArqUserLibrary.m

@@ -1,40 +0,0 @@
-/*
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
-
-#import "ArqUserLibrary.h"
-
-
-@implementation ArqUserLibrary
-+ (NSString *)arqCachesPath {
-    return [NSHomeDirectory() stringByAppendingPathComponent:@"/Library/Arq/Caches.noindex"];
-}
-@end

ArqVerifyCommand.h

@@ -14,8 +14,11 @@
     NSString *secretKey;
     NSString *encryptionPassword;
     S3Service *s3;
+    NSSet *objectSHA1s;
+    BOOL verbose;
 }
 - (id)initWithAccessKey:(NSString *)theAccessKey secretKey:(NSString *)theSecretKey encryptionPassword:(NSString *)theEncryptionPassword;
+- (void)setVerbose:(BOOL)isVerbose;
 - (BOOL)verifyAll:(NSError **)error;
 - (BOOL)verifyS3BucketName:(NSString *)s3BucketName error:(NSError **)error;
 - (BOOL)verifyS3BucketName:(NSString *)s3BucketName computerUUID:(NSString *)computerUUID error:(NSError **)error;

ArqVerifyCommand.m

@@ -14,9 +14,11 @@
 #import "BucketVerifier.h"
 #import "NSError_extra.h"
 #import "NSErrorCodes.h"
+#import "ArqSalt.h"
+#import "ArqRepo.h"
 
 @interface ArqVerifyCommand (internal)
-- (NSArray *)objectSHA1sForS3BucketName:(NSString *)s3BucketName computerUUID:(NSString *)computerUUID error:(NSError **)error;
+- (BOOL)loadObjectSHA1sForS3BucketName:(NSString *)s3BucketName computerUUID:(NSString *)computerUUID error:(NSError **)error;
 @end
 
 @implementation ArqVerifyCommand
@@ -26,7 +28,7 @@
 		secretKey = [theSecretKey retain];
 		encryptionPassword = [theEncryptionPassword retain];
 		S3AuthorizationProvider *sap = [[S3AuthorizationProvider alloc] initWithAccessKey:accessKey secretKey:secretKey];
-		s3 = [[S3Service alloc] initWithS3AuthorizationProvider:sap useSSL:NO retryOnNetworkError:YES];
+		s3 = [[S3Service alloc] initWithS3AuthorizationProvider:sap useSSL:NO retryOnTransientError:YES];
 		[sap release];
     }
     return self;
@@ -38,6 +40,9 @@
     [s3 release];
     [super dealloc];
 }
+- (void)setVerbose:(BOOL)isVerbose {
+    verbose = isVerbose;
+}
 - (BOOL)verifyAll:(NSError **)error {
 	NSArray *s3BucketNames = [S3Service s3BucketNamesForAccessKeyID:accessKey];
 	for (NSString *s3BucketName in s3BucketNames) {
@@ -87,52 +92,82 @@
 	return YES;
 }
 - (BOOL)verifyS3BucketName:(NSString *)s3BucketName computerUUID:(NSString *)computerUUID error:(NSError **)error {
-	printf("verifying computerUUID %s s3Bucket %s\n", [computerUUID UTF8String], [s3BucketName UTF8String]);
+	printf("\nverifying computerUUID %s s3Bucket %s\n", [computerUUID UTF8String], [s3BucketName UTF8String]);
 	NSString *computerBucketsPrefix = [NSString stringWithFormat:@"/%@/%@/buckets", s3BucketName, computerUUID];
 	NSArray *s3BucketUUIDPaths = [s3 pathsWithPrefix:computerBucketsPrefix error:error];
 	if (s3BucketUUIDPaths == nil) {
 		return NO;
 	}
-	NSArray *objectSHA1s = [self objectSHA1sForS3BucketName:s3BucketName computerUUID:computerUUID error:error];
+    
-	if (objectSHA1s == nil) {
+    NSMutableArray *bucketUUIDs = [NSMutableArray array];
-		return NO;
-	}
 	for (NSString *s3BucketUUIDPath in s3BucketUUIDPaths) {
-		NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
+        NSString *bucketUUID = [s3BucketUUIDPath lastPathComponent];
-		NSString *bucketUUID = [s3BucketUUIDPath lastPathComponent];
+        printf("found bucket UUID %s\n", [bucketUUID UTF8String]);
-		printf("verifying bucketUUID %s computerUUID %s s3Bucket %s\n", [bucketUUID UTF8String], [computerUUID UTF8String], [s3BucketName UTF8String]);
+        [bucketUUIDs addObject:bucketUUID];
-		BucketVerifier *bucketVerifier = [[[BucketVerifier alloc] initWithS3Service:s3
+    }
-																	   s3BucketName:s3BucketName
+    
-																	   computerUUID:computerUUID
+    [objectSHA1s release];
-																		 bucketUUID:bucketUUID
+    objectSHA1s = nil;
-																	  s3ObjectSHA1s:objectSHA1s
+    if (![self loadObjectSHA1sForS3BucketName:s3BucketName computerUUID:computerUUID error:error]) {
-																	  encryptionKey:encryptionPassword] autorelease];
+        return NO;
-		BOOL ret = [bucketVerifier verify:error];
+    }
-		if (error != NULL) {
+    
-			[*error retain];
+    BOOL ret = YES;
-		}
+    NSAutoreleasePool *pool = nil;
-		[pool drain];
+    for (NSString *bucketUUID in bucketUUIDs) {
-		if (error != NULL) {
+        [pool release];
-			[*error autorelease];
+        pool = [[NSAutoreleasePool alloc] init];
-		}
+        if (![self verifyS3BucketName:s3BucketName computerUUID:computerUUID bucketUUID:bucketUUID error:error]) {
-		if (!ret) {
+            ret = NO;
-			return NO;
+            break;
-		}
+        }
-	}
+    }
-	return YES;
+    if (!ret && error != NULL) {
+        [*error retain];
+    }
+    [pool drain];
+    if (!ret && error != NULL) {
+        [*error autorelease];
+    }
+    return ret;
 }
 - (BOOL)verifyS3BucketName:(NSString *)s3BucketName computerUUID:(NSString *)computerUUID bucketUUID:(NSString *)bucketUUID error:(NSError **)error {
-	NSArray *objectSHA1s = [self objectSHA1sForS3BucketName:s3BucketName computerUUID:computerUUID error:error];
+    if (objectSHA1s == nil) {
-	if (objectSHA1s == nil) {
+        if (![self loadObjectSHA1sForS3BucketName:s3BucketName computerUUID:computerUUID error:error]) {
-		return NO;
+            return NO;
-	}
+        }
-	printf("verifying bucketUUID %s computerUUID %s s3Bucket %s\n", [bucketUUID UTF8String], [computerUUID UTF8String], [s3BucketName UTF8String]);
+    }
+    
+    NSError *saltError = nil;
+    ArqSalt *arqSalt = [[[ArqSalt alloc] initWithAccessKeyID:accessKey secretAccessKey:secretKey s3BucketName:s3BucketName computerUUID:computerUUID] autorelease];
+    NSData *salt = [arqSalt salt:&saltError];
+    if (salt == nil) {
+        if ([saltError code] != ERROR_NOT_FOUND) {
+            if (error != NULL) {
+                *error = saltError;
+            }
+            return NO;
+        }
+    }
+    ArqRepo *repo = [[[ArqRepo alloc] initWithS3Service:s3
+                                           s3BucketName:s3BucketName
+                                           computerUUID:computerUUID
+                                             bucketUUID:bucketUUID
+                                     encryptionPassword:encryptionPassword
+                                                   salt:salt
+                                                  error:error] autorelease];
+    if (repo == nil) {
+        return NO;
+    }
+    printf("\nverifying bucketUUID %s computerUUID %s s3Bucket %s\n", [bucketUUID UTF8String], [computerUUID UTF8String], [s3BucketName UTF8String]);
+    
 	BucketVerifier *bucketVerifier = [[[BucketVerifier alloc] initWithS3Service:s3
 																   s3BucketName:s3BucketName
 																   computerUUID:computerUUID
 																	 bucketUUID:bucketUUID
 																  s3ObjectSHA1s:objectSHA1s
-																  encryptionKey:encryptionPassword] autorelease];
+                                                                        verbose:verbose
+                                                                           repo:repo] autorelease];
 	if (![bucketVerifier verify:error]) {
 		return NO;
 	}
@@ -141,27 +176,19 @@
 @end
 
 @implementation ArqVerifyCommand (internal)
-- (NSArray *)objectSHA1sForS3BucketName:(NSString *)s3BucketName computerUUID:(NSString *)computerUUID error:(NSError **)error {
+- (BOOL)loadObjectSHA1sForS3BucketName:(NSString *)s3BucketName computerUUID:(NSString *)computerUUID error:(NSError **)error {
-	NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
+	NSMutableSet *theObjectSHA1s = [NSMutableSet set];
-	NSMutableArray *objectSHA1s = nil;
 	NSString *objectsPrefix = [NSString stringWithFormat:@"/%@/%@/objects", s3BucketName, computerUUID];
 	printf("loading S3 object SHA1s with prefix %s\n", [objectsPrefix UTF8String]);
 	NSArray *objectPaths = [s3 pathsWithPrefix:objectsPrefix error:error];
-	if (objectPaths != nil) {
+    if (objectPaths == nil) {
-		objectSHA1s = [[NSMutableArray alloc] init];
+        return NO;
-		printf("loaded %u object SHA1s with prefix %s\n", [objectPaths count], [objectsPrefix UTF8String]);
+    }
-		for (NSString *objectPath in objectPaths) {
+    for (NSString *objectPath in objectPaths) {
-			[objectSHA1s addObject:[objectPath lastPathComponent]];
+        [theObjectSHA1s addObject:[objectPath lastPathComponent]];
-		}
+    }
-	}
+    objectSHA1s = [theObjectSHA1s retain];
-	if (error != NULL) {
+    printf("loaded %u object SHA1s with prefix %s\n", [objectSHA1s count], [objectsPrefix UTF8String]);
-		[*error retain];
+    return YES;
-	}
-	[pool drain];
-	[objectSHA1s autorelease];
-	if (error != NULL) {
-		[*error autorelease];
-	}
-	return objectSHA1s;
 }
 @end

BlobKey.h

@@ -0,0 +1,19 @@
+//
+//  BlobKey.h
+//  Arq
+//
+//  Created by Stefan Reitshamer on 6/27/11.
+//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#import <Cocoa/Cocoa.h>
+
+
+@interface BlobKey : NSObject <NSCopying> {
+    NSString *sha1;
+    BOOL stretchEncryptionKey;
+}
+- (id)initWithSHA1:(NSString *)theSHA1 stretchEncryptionKey:(BOOL)isStretchedKey;
+- (NSString *)sha1;
+- (BOOL)stretchEncryptionKey;
+@end

BlobKey.m

@@ -0,0 +1,53 @@
+//
+//  BlobKey.m
+//  Arq
+//
+//  Created by Stefan Reitshamer on 6/27/11.
+//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#import "BlobKey.h"
+
+
+@implementation BlobKey
+- (id)initWithSHA1:(NSString *)theSHA1 stretchEncryptionKey:(BOOL)isStretchedKey {
+    if (self = [super init]) {
+        sha1 = [theSHA1 retain];
+        stretchEncryptionKey = isStretchedKey;
+    }
+    return self;
+}
+- (void)dealloc {
+    [sha1 release];
+    [super dealloc];
+}
+
+- (NSString *)sha1 {
+    return sha1;
+}
+- (BOOL)stretchEncryptionKey {
+    return stretchEncryptionKey;
+}
+
+
+#pragma mark NSCopying
+- (id)copyWithZone:(NSZone *)zone {
+    return [[BlobKey alloc] initWithSHA1:sha1 stretchEncryptionKey:stretchEncryptionKey];
+}
+
+
+#pragma mark NSObject
+- (NSString *)description {
+    return [NSString stringWithFormat:@"<BlobKey %@,stretchedkey=%@>", sha1, (stretchEncryptionKey ? @"YES" : @"NO")];
+}
+- (BOOL)isEqual:(id)anObject {
+    if (![anObject isKindOfClass:[BlobKey class]]) {
+        return NO;
+    }
+    BlobKey *other = (BlobKey *)anObject;
+    return [[other sha1] isEqualToString:sha1] && [other stretchEncryptionKey] == stretchEncryptionKey;
+}
+- (NSUInteger)hash {
+    return [sha1 hash] + (stretchEncryptionKey ? 1 : 0);
+}
+@end

Commit.h

@@ -1,62 +1,54 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  Commit.h
- 
+//  Backup
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 3/21/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 PhotoMinds LLC. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #import <Cocoa/Cocoa.h>
 #import "Blob.h"
 #import "BufferedInputStream.h"
+@class BlobKey;
 
-#define CURRENT_COMMIT_VERSION 3
+#define CURRENT_COMMIT_VERSION 4
 
 @interface Commit : NSObject {
     int commitVersion;
 	NSString *_author;
 	NSString *_comment;
-	NSMutableSet *_parentCommitSHA1s;
+	NSMutableSet *_parentCommitBlobKeys;
-	NSString *_treeSHA1;
+	BlobKey *_treeBlobKey;
 	NSString *_location;
     NSString *_computer;
-	NSString *_mergeCommonAncestorCommitSHA1;
+	BlobKey *_mergeCommonAncestorCommitBlobKey;
 	NSDate *_creationDate;
     NSArray *_commitFailedFiles;
 }
++ (NSString *)errorDomain;
+- (id)initWithCommit:(Commit *)commit parentCommitBlobKey:(BlobKey *)parentCommitBlobKey;
+
+- (id)             initWithAuthor:(NSString *)theAuthor 
+                          comment:(NSString *)theComment 
+             parentCommitBlobKeys:(NSSet *)theParentCommitBlobKeys 
+                      treeBlobKey:(BlobKey *)theTreeBlobKey
+                         location:(NSString *)theLocation
+ mergeCommonAncestorCommitBlobKey:(BlobKey *)theMergeCommonAncestorCommitBlobKey
+                commitFailedFiles:(NSArray *)theCommitFailedFiles;
+
 - (id)initWithBufferedInputStream:(BufferedInputStream *)is error:(NSError **)error;
 
 @property(readonly,copy) NSString *author;
 @property(readonly,copy) NSString *comment;
-@property(readonly,copy) NSString *treeSHA1;
+@property(readonly,copy) BlobKey *treeBlobKey;
-@property(readonly,retain) NSSet *parentCommitSHA1s;
+@property(readonly,retain) NSSet *parentCommitBlobKeys;
 @property(readonly,copy) NSString *location;
 @property(readonly,copy) NSString *computer;
-@property(readonly,copy) NSString *mergeCommonAncestorCommitSHA1;
+@property(readonly,copy) BlobKey *mergeCommonAncestorCommitBlobKey;
 @property(readonly,retain) NSDate *creationDate;
 @property(readonly,retain) NSArray *commitFailedFiles;
+
+- (NSNumber *)isMergeCommit;
+- (Blob *)toBlob;
+
 @end

Commit.m

@@ -1,45 +1,24 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  Commit.m
- 
+//  Backup
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 3/21/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 PhotoMinds LLC. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #import "IntegerIO.h"
 #import "DateIO.h"
 #import "StringIO.h"
 #import "Commit.h"
+#import "Blob.h"
 #import "DataInputStream.h"
-#import "BufferedInputStream.h"
 #import "RegexKitLite.h"
 #import "SetNSError.h"
 #import "NSErrorCodes.h"
 #import "CommitFailedFile.h"
+#import "BufferedInputStream.h"
+#import "BooleanIO.h"
+#import "BlobKey.h"
 
 #define HEADER_LENGTH (10)
 
@@ -48,97 +27,170 @@
 @end
 
 @implementation Commit
++ (NSString *)errorDomain {
+    return @"CommitErrorDomain";
+}
+
+
 @synthesize author = _author, 
 comment = _comment, 
-treeSHA1 = _treeSHA1, 
+treeBlobKey = _treeBlobKey, 
-parentCommitSHA1s = _parentCommitSHA1s, 
+parentCommitBlobKeys = _parentCommitBlobKeys, 
 location = _location, 
 computer = _computer,
-mergeCommonAncestorCommitSHA1 = _mergeCommonAncestorCommitSHA1, 
+mergeCommonAncestorCommitBlobKey = _mergeCommonAncestorCommitBlobKey, 
 creationDate = _creationDate,
 commitFailedFiles = _commitFailedFiles;
 
+
+- (id)initWithCommit:(Commit *)theCommit parentCommitBlobKey:(BlobKey *)theParentBlobKey {
+    if (self = [super init]) {
+        _author = [[theCommit author] copy];
+        _comment = [[theCommit comment] copy];
+        if (theParentBlobKey != nil) {
+            _parentCommitBlobKeys = [[NSSet alloc] initWithObjects:theParentBlobKey, nil];
+        } else {
+            _parentCommitBlobKeys = [[NSSet alloc] init];
+        }
+        _treeBlobKey = [[theCommit treeBlobKey] copy];
+        _location = [[theCommit location] copy];
+        _computer = [[theCommit computer] copy];
+        _mergeCommonAncestorCommitBlobKey = nil;
+        _creationDate = [[theCommit creationDate] copy];
+        _commitFailedFiles = [[theCommit commitFailedFiles] copy];
+    }
+    return self;
+}
+
+- (id)             initWithAuthor:(NSString *)theAuthor 
+                          comment:(NSString *)theComment 
+             parentCommitBlobKeys:(NSSet *)theParentCommitBlobKeys 
+                      treeBlobKey:(BlobKey *)theTreeBlobKey
+                         location:(NSString *)theLocation
+ mergeCommonAncestorCommitBlobKey:(BlobKey *)theMergeCommonAncestorCommitBlobKey 
+                commitFailedFiles:(NSArray *)theCommitFailedFiles {
+    if (self = [super init]) {
+        _author = [theAuthor copy];
+        _comment = [theComment copy];
+        _parentCommitBlobKeys = [[NSSet alloc] initWithSet:theParentCommitBlobKeys];
+        _treeBlobKey = [theTreeBlobKey retain];
+        _location = [theLocation copy];
+        NSRange computerRange = [_location rangeOfRegex:@"^file://([^/]+)/" capture:1];
+        if (computerRange.location != NSNotFound) {
+            _computer = [[_location substringWithRange:computerRange] stringByReplacingPercentEscapesUsingEncoding:NSUTF8StringEncoding];
+        } else {
+            _computer = @"";
+        }
+        [_computer retain];
+        _mergeCommonAncestorCommitBlobKey = [theMergeCommonAncestorCommitBlobKey retain];
+        _creationDate = [[NSDate alloc] init];
+        _commitFailedFiles = [theCommitFailedFiles copy];
+    }
+    return self;
+}
 - (id)initWithBufferedInputStream:(BufferedInputStream *)is error:(NSError **)error {
 	if (self = [super init]) {
-        _parentCommitSHA1s = [[NSMutableSet alloc] init];
+        _parentCommitBlobKeys = [[NSMutableSet alloc] init];
         if (![self readHeader:is error:error]) {
-            [self release];
+            goto init_error;
-            return nil;
         }
-        BOOL ret = NO;
+        if (![StringIO read:&_author from:is error:error]) {
-        do {
+            goto init_error;
-            if (![StringIO read:&_author from:is error:error]) {
+        }
-                break;
+        [_author retain];
+        
+        if (![StringIO read:&_comment from:is error:error]) {
+            goto init_error;
+        }
+        [_comment retain];
+        
+        uint64_t parentCommitKeyCount = 0;
+        if (![IntegerIO readUInt64:&parentCommitKeyCount from:is error:error]) {
+            goto init_error;
+        }
+        for (uint64_t i = 0; i < parentCommitKeyCount; i++) {
+            NSString *key;
+            BOOL cryptoKeyStretched = NO;
+            if (![StringIO read:&key from:is error:error]) {
+                goto init_error;
             }
-            [_author retain];
+            if (commitVersion >= 4) {
-            
+                if (![BooleanIO read:&cryptoKeyStretched from:is error:error]) {
-            if (![StringIO read:&_comment from:is error:error]) {
+                    goto init_error;
-                break;
-            }
-            [_comment retain];
-            
-            uint64_t parentCommitKeyCount = 0;
-            if (![IntegerIO readUInt64:&parentCommitKeyCount from:is error:error]) {
-                break;
-            }
-            for (uint64_t i = 0; i < parentCommitKeyCount; i++) {
-                NSString *key;
-                if (![StringIO read:&key from:is error:error]) {
-                    break;
                 }
-                [_parentCommitSHA1s addObject:key];
             }
-            
+            BlobKey *parentBlobKey = [[BlobKey alloc] initWithSHA1:key stretchEncryptionKey:cryptoKeyStretched];
-            if (![StringIO read:&_treeSHA1 from:is error:error]) {
+            [_parentCommitBlobKeys addObject:parentBlobKey];
-                break;
+            [parentBlobKey release];
+        }
+        
+        NSString *treeSHA1 = nil;
+        BOOL treeStretchedKey = NO;
+        if (![StringIO read:&treeSHA1 from:is error:error]) {
+            goto init_error;
+        }
+        if (commitVersion >= 4) {
+            if (![BooleanIO read:&treeStretchedKey from:is error:error]) {
+                goto init_error;
             }
-            [_treeSHA1 retain];
+        }
-            
+        _treeBlobKey = [[BlobKey alloc] initWithSHA1:treeSHA1 stretchEncryptionKey:treeStretchedKey];
-            if (![StringIO read:&_location from:is error:error]) {
+        
-                break;
+        if (![StringIO read:&_location from:is error:error]) {
+            goto init_error;
+        }
+        [_location retain];
+        
+        NSRange computerRange = [_location rangeOfRegex:@"^file://([^/]+)/" capture:1];
+        if (computerRange.location != NSNotFound) {
+            _computer = [[_location substringWithRange:computerRange] stringByReplacingPercentEscapesUsingEncoding:NSUTF8StringEncoding];
+        } else {
+            _computer = @"";
+        }
+        [_computer retain];
+        
+        NSString *mergeCommonAncestorCommitSHA1 = nil;
+        BOOL mergeCommonAncestorCommitStretchedKey = NO;
+        if (![StringIO read:&mergeCommonAncestorCommitSHA1 from:is error:error]) {
+            goto init_error;
+        }
+        if (commitVersion >= 4) {
+            if (![BooleanIO read:&mergeCommonAncestorCommitStretchedKey from:is error:error]) {
+                goto init_error;
             }
-            [_location retain];
+        }
-            
+        if (mergeCommonAncestorCommitSHA1 != nil) {
-            NSRange computerRange = [_location rangeOfRegex:@"^file://([^/]+)/" capture:1];
+            _mergeCommonAncestorCommitBlobKey = [[BlobKey alloc] initWithSHA1:mergeCommonAncestorCommitSHA1 stretchEncryptionKey:mergeCommonAncestorCommitStretchedKey];
-            if (computerRange.location != NSNotFound) {
+        }
-                _computer = [[_location substringWithRange:computerRange] stringByReplacingPercentEscapesUsingEncoding:NSUTF8StringEncoding];
+        
-            } else {
+        if (![DateIO read:&_creationDate from:is error:error]) {
-                _computer = @"";
+            goto init_error;
+        }
+        [_creationDate retain];
+        if (commitVersion >= 3) {
+            uint64_t commitFailedFileCount = 0;
+            if (![IntegerIO readUInt64:&commitFailedFileCount from:is error:error]) {
+                goto init_error;
             }
-            [_computer retain];
+            NSMutableArray *commitFailedFiles = [NSMutableArray array];
-            
+            for (uint64_t index = 0; index < commitFailedFileCount; index++) {
-            if (![StringIO read:&_mergeCommonAncestorCommitSHA1 from:is error:error]) {
+                CommitFailedFile *cff = [[CommitFailedFile alloc] initWithInputStream:is error:error];
-                break;
+                if (cff == nil) {
-            }
+                    goto init_error;
-            [_mergeCommonAncestorCommitSHA1 retain];
-            
-            if (![DateIO read:&_creationDate from:is error:error]) {
-                break;
-            }
-            [_creationDate retain];
-            if (commitVersion >= 3) {
-                uint64_t commitFailedFileCount = 0;
-                if (![IntegerIO readUInt64:&commitFailedFileCount from:is error:error]) {
-                    break;
                 }
-                NSMutableArray *commitFailedFiles = [NSMutableArray array];
+                [commitFailedFiles addObject:cff];
-                for (uint64_t index = 0; index < commitFailedFileCount; index++) {
+                [cff release];
-                    CommitFailedFile *cff = [[CommitFailedFile alloc] initWithInputStream:is error:error];
-                    if (cff == nil) {
-                        break;
-                    }
-                    [commitFailedFiles addObject:cff];
-                    [cff release];
-                }
-                _commitFailedFiles = [commitFailedFiles retain];
             }
-            ret = YES;
+            _commitFailedFiles = [commitFailedFiles retain];
-        } while (0);
-        if (!ret) {
-            [self release];
-            self = nil;
         }
     }
+    goto init_done;
+    
+init_error:
+    [self release];
+    self = nil;
+    
+init_done:
     return self;
 }
 - (void)release {
@@ -147,24 +199,62 @@ commitFailedFiles = _commitFailedFiles;
 - (void)dealloc {
     [_author release];
     [_comment release];
-    [_parentCommitSHA1s release];
+    [_parentCommitBlobKeys release];
-    [_treeSHA1 release];
+    [_treeBlobKey release];
     [_location release];
     [_computer release];
-    [_mergeCommonAncestorCommitSHA1 release];
+    [_mergeCommonAncestorCommitBlobKey release];
     [_creationDate release];
     [_commitFailedFiles release];
     [super dealloc];
 }
+- (NSNumber *)isMergeCommit {
+    return [NSNumber numberWithBool:([_parentCommitBlobKeys count] > 1)];
+}
+- (Blob *)toBlob {
+    Blob *ret = nil;
+    NSMutableData *data = [[NSMutableData alloc] init];
+    char header[HEADER_LENGTH + 1];
+    sprintf(header, "CommitV%03d", CURRENT_COMMIT_VERSION);
+    [data appendBytes:header length:HEADER_LENGTH];
+    [StringIO write:_author to:data];
+    [StringIO write:_comment to:data];
+    uint64_t parentCommitBlobKeysCount = (uint64_t)[_parentCommitBlobKeys count];
+    [IntegerIO writeUInt64:parentCommitBlobKeysCount to:data];
+    for (BlobKey *parentCommitBlobKey in _parentCommitBlobKeys) {
+        [StringIO write:[parentCommitBlobKey sha1] to:data];
+        [BooleanIO write:[parentCommitBlobKey stretchEncryptionKey] to:data];
+    }
+    [StringIO write:[_treeBlobKey sha1] to:data];
+    [BooleanIO write:[_treeBlobKey stretchEncryptionKey] to:data];
+    [StringIO write:_location to:data];
+    [StringIO write:[_mergeCommonAncestorCommitBlobKey sha1] to:data];
+    [BooleanIO write:[_mergeCommonAncestorCommitBlobKey stretchEncryptionKey] to:data];
+    [DateIO write:_creationDate to:data];
+    uint64_t commitFailedFilesCount = (uint64_t)[_commitFailedFiles count];
+    [IntegerIO writeUInt64:commitFailedFilesCount to:data];
+    for (CommitFailedFile *cff in _commitFailedFiles) {
+        [cff writeTo:data];
+    }
+    ret = [[[Blob alloc] initWithData:data mimeType:@"binary/octet-stream" downloadName:@"commit" dataDescription:@"commit"] autorelease];
+    [data release];
+    return ret;
+}
+
+#pragma mark NSObject
+- (NSString *)description {
+    return [NSString stringWithFormat:@"<Commit: created=%@ tree=%@ parents=%@>", _creationDate, _treeBlobKey, _parentCommitBlobKeys];
+}
 @end
 
 @implementation Commit (internal)
 - (BOOL)readHeader:(BufferedInputStream *)is error:(NSError **)error {
-    NSData *headerData = [is readExactly:HEADER_LENGTH error:error];
+    BOOL ret = NO;
-    if (headerData == nil) {
+    unsigned char *buf = (unsigned char *)malloc(HEADER_LENGTH);
-        return NO;
+    if (![is readExactly:HEADER_LENGTH into:buf error:error]) {
+        goto readHeader_error;
     }
-    NSString *header = [[[NSString alloc] initWithData:headerData encoding:NSUTF8StringEncoding] autorelease];
+    NSString *header = [[[NSString alloc] initWithBytes:buf length:HEADER_LENGTH encoding:NSASCIIStringEncoding] autorelease];
     NSRange versionRange = [header rangeOfRegex:@"^CommitV(\\d{3})$" capture:1];
     commitVersion = 0;
     if (versionRange.location != NSNotFound) {
@@ -174,10 +264,12 @@ commitFailedFiles = _commitFailedFiles;
         [nf release];
     }
     if (commitVersion > CURRENT_COMMIT_VERSION || commitVersion < 2) {
-        SETNSERROR(@"TreeErrorDomain", ERROR_INVALID_OBJECT_VERSION, @"invalid Commit header");
+        SETNSERROR([Commit errorDomain], ERROR_INVALID_OBJECT_VERSION, @"invalid Commit header");
-        return NO;
+        goto readHeader_error;
     }
-    return YES;
+    ret = YES;
-    
+readHeader_error:
+    free(buf);
+    return ret;
 }
 @end

CommitFailedFile.h

@@ -1,44 +1,22 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  CommitFailedFile.h
- 
+//  Arq
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 2/22/10.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2010 __MyCompanyName__. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #import <Cocoa/Cocoa.h>
 @class BufferedInputStream;
 
 @interface CommitFailedFile : NSObject {
-    NSString *relativePath;
+    NSString *path;
     NSString *errorMessage;
 }
+- (id)initWithPath:(NSString *)thePath errorMessage:(NSString *)theErrorMessage;
 - (id)initWithInputStream:(BufferedInputStream *)is error:(NSError **)error;
-- (NSString *)relativePath;
+- (NSString *)path;
 - (NSString *)errorMessage;
 - (void)writeTo:(NSMutableData *)data;
+- (BOOL)isEqualToCommitFailedFile:(CommitFailedFile *)cff;
 @end

CommitFailedFile.m

@@ -1,65 +1,62 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  CommitFailedFile.m
- 
+//  Arq
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 2/22/10.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2010 __MyCompanyName__. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #import "CommitFailedFile.h"
 #import "StringIO.h"
 #import "BufferedInputStream.h"
 
 @implementation CommitFailedFile
+- (id)initWithPath:(NSString *)thePath errorMessage:(NSString *)theErrorMessage {
+    if (self = [super init]) {
+        path = [thePath copy];
+        errorMessage = [theErrorMessage copy];
+    }
+    return self;
+}
 - (id)initWithInputStream:(BufferedInputStream *)is error:(NSError **)error {
     if (self = [super init]) {
-        if (![StringIO read:&relativePath from:is error:error]
+        if (![StringIO read:&path from:is error:error]
             || ![StringIO read:&errorMessage from:is error:error]) {
             [self release];
-            self = nil;
+            return nil;
         }
-        [relativePath retain];
+        [path retain];
         [errorMessage retain];
     }
     return self;
 }
 - (void)dealloc {
-    [relativePath release];
+    [path release];
     [errorMessage release];
     [super dealloc];
 }
-- (NSString *)relativePath {
+- (NSString *)path {
-    return [[relativePath retain] autorelease];
+    return [[path retain] autorelease];
 }
 - (NSString *)errorMessage {
     return [[errorMessage retain] autorelease];
 }
 - (void)writeTo:(NSMutableData *)data {
-    [StringIO write:relativePath to:data];
+    [StringIO write:path to:data];
     [StringIO write:errorMessage to:data];
 }
+- (BOOL)isEqualToCommitFailedFile:(CommitFailedFile *)cff {
+    return [[cff path] isEqualToString:path] && [[cff errorMessage] isEqualToString:errorMessage];
+}
+
+#pragma mark NSObject
+- (BOOL)isEqual:(id)other {
+    if (other == self) {
+        return YES;
+    }
+    if (other == nil || ![other isKindOfClass:[self class]]) {
+        return NO;
+    }
+    return [self isEqualToCommitFailedFile:other];
+}
 @end

DiskPack.h

@@ -1,34 +1,10 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  DiskPack.h
- 
+//  Arq
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 12/30/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 __MyCompanyName__. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #import <Cocoa/Cocoa.h>
 @class S3Service;
@@ -42,12 +18,22 @@
     NSString *packSHA1;
     NSString *s3Path;
     NSString *localPath;
+    uid_t targetUID;
+    gid_t targetGID;
 }
 + (NSString *)s3PathWithS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1;
-+ (NSString *)localPathWithComputerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1;
++ (NSString *)localPathWithS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1;
-- (id)initWithS3Service:(S3Service *)theS3 s3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1;
+- (id)initWithS3Service:(S3Service *)theS3 
+           s3BucketName:(NSString *)theS3BucketName 
+           computerUUID:(NSString *)theComputerUUID 
+            packSetName:(NSString *)thePackSetName 
+               packSHA1:(NSString *)thePackSHA1
+              targetUID:(uid_t)theTargetUID 
+              targetGID:(gid_t)theTargetGID;
 - (BOOL)makeLocal:(NSError **)error;
+- (BOOL)makeNotLocal:(NSError **)error;
 - (ServerBlob *)newServerBlobForObjectAtOffset:(unsigned long long)offset error:(NSError **)error;
 - (BOOL)fileLength:(unsigned long long *)length error:(NSError **)error;
+- (BOOL)copyToPath:(NSString *)dest error:(NSError **)error;
 - (NSArray *)sortedPackIndexEntries:(NSError **)error;
 @end

DiskPack.m

@@ -1,40 +1,15 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  DiskPack.m
- 
+//  Arq
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 12/30/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 __MyCompanyName__. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #include <sys/stat.h>
 #import "DiskPack.h"
 #import "SetNSError.h"
 #import "FDInputStream.h"
-#import "BufferedInputStream.h"
 #import "StringIO.h"
 #import "IntegerIO.h"
 #import "ServerBlob.h"
@@ -48,10 +23,13 @@
 #import "S3ObjectMetadata.h"
 #import "PackIndexEntry.h"
 #import "SHA1Hash.h"
-#import "ArqUserLibrary.h"
+#import "UserLibrary_Arq.h"
+#import "BufferedInputStream.h"
+#import "NSError_extra.h"
+
 
 @interface DiskPack (internal)
-- (BOOL)savePack:(ServerBlob *)sb bytesWritten:(unsigned long long *)written error:(NSError **)error;
+- (BOOL)savePack:(ServerBlob *)sb error:(NSError **)error;
 - (NSArray *)sortedPackIndexEntriesFromStream:(BufferedInputStream *)fis error:(NSError **)error;
 @end
 
@@ -59,10 +37,16 @@
 + (NSString *)s3PathWithS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1 {
     return [NSString stringWithFormat:@"/%@/%@/packsets/%@/%@.pack", theS3BucketName, theComputerUUID, thePackSetName, thePackSHA1];
 }
-+ (NSString *)localPathWithComputerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1 {
++ (NSString *)localPathWithS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1 {
-    return [NSString stringWithFormat:@"%@/%@/packsets/%@/%@/%@.pack", [ArqUserLibrary arqCachesPath], theComputerUUID, thePackSetName, [thePackSHA1 substringToIndex:2], [thePackSHA1 substringFromIndex:2]];
+    return [NSString stringWithFormat:@"%@/%@/%@/packsets/%@/%@/%@.pack", [UserLibrary arqCachePath], theS3BucketName, theComputerUUID, thePackSetName, [thePackSHA1 substringToIndex:2], [thePackSHA1 substringFromIndex:2]];
 }
-- (id)initWithS3Service:(S3Service *)theS3 s3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1 {
+- (id)initWithS3Service:(S3Service *)theS3 
+           s3BucketName:(NSString *)theS3BucketName 
+           computerUUID:(NSString *)theComputerUUID 
+            packSetName:(NSString *)thePackSetName 
+               packSHA1:(NSString *)thePackSHA1
+              targetUID:(uid_t)theTargetUID 
+              targetGID:(gid_t)theTargetGID {
     if (self = [super init]) {
         s3 = [theS3 retain];
         s3BucketName = [theS3BucketName retain];
@@ -70,7 +54,9 @@
         packSetName = [thePackSetName retain];
         packSHA1 = [thePackSHA1 retain];
         s3Path = [[DiskPack s3PathWithS3BucketName:s3BucketName computerUUID:computerUUID packSetName:packSetName packSHA1:packSHA1] retain];
-        localPath = [[DiskPack localPathWithComputerUUID:computerUUID packSetName:packSetName packSHA1:packSHA1] retain];
+        localPath = [[DiskPack localPathWithS3BucketName:s3BucketName computerUUID:computerUUID packSetName:packSetName packSHA1:packSHA1] retain];
+        targetUID = theTargetUID;
+        targetGID = theTargetGID;
     }
     return self;
 }
@@ -86,38 +72,59 @@
 }
 - (BOOL)makeLocal:(NSError **)error {
     NSFileManager *fm = [NSFileManager defaultManager];
-    BOOL ret = NO;
+    BOOL ret = YES;
     if (![fm fileExistsAtPath:localPath]) {
-        HSLogDebug(@"packset %@: making pack %@ local", packSetName, packSHA1);
+        for (;;) {
-        NSError *myError = nil;
+            HSLogDebug(@"packset %@: making pack %@ local", packSetName, packSHA1);
-        ServerBlob *sb = [s3 newServerBlobAtPath:s3Path error:&myError];
+            NSError *myError = nil;
-        if (sb == nil) {
+            ServerBlob *sb = [s3 newServerBlobAtPath:s3Path error:&myError];
-            HSLogError(@"error getting S3 pack %@: %@", s3Path, [myError localizedDescription]);
+            if (sb != nil) {
-            if (error != NULL) {
+                ret = [self savePack:sb error:error];
-                *error = myError;
+                [sb release];
+                break;
+            }
+            if (![myError isTransientError]) {
+                HSLogError(@"error getting S3 pack %@: %@", s3Path, myError);
+                if (error != NULL) {
+                    *error = myError;
+                }
+                ret = NO;
+                break;
+            }
+            HSLogWarn(@"network error making pack %@ local (retrying): %@", s3Path, myError);
+            NSError *rmError = nil;
+            if (![[NSFileManager defaultManager] removeItemAtPath:localPath error:&rmError]) {
+                HSLogError(@"error deleting incomplete downloaded pack file %@: %@", localPath, rmError);
             }
-        } else {
-            unsigned long long bytesWritten;
-            ret = [self savePack:sb bytesWritten:&bytesWritten error:error];
-            [sb release];
         }
-    } else {
+    }
-        ret = YES;
+    return ret;
+}
+- (BOOL)makeNotLocal:(NSError **)error {
+    NSFileManager *fm = [NSFileManager defaultManager];
+    HSLogDebug(@"removing disk pack %@", localPath);
+    BOOL ret = YES;
+    if ([fm fileExistsAtPath:localPath] && ![fm removeItemAtPath:localPath error:error]) {
+        ret = NO;
     }
     return ret;
 }
 - (ServerBlob *)newServerBlobForObjectAtOffset:(unsigned long long)offset error:(NSError **)error {
     int fd = open([localPath fileSystemRepresentation], O_RDONLY);
     if (fd == -1) {
-        SETNSERROR(@"UnixErrorDomain", errno, @"%s: %@", strerror(errno), localPath);
+        int errnum = errno;
+        HSLogError(@"open(%@) error %d: %s", localPath, errnum, strerror(errnum));
+        SETNSERROR(@"UnixErrorDomain", errnum, @"failed to open %@: %s", localPath, strerror(errnum));
         return nil;
     }
     ServerBlob *ret = nil;
-    FDInputStream *fdis = [[FDInputStream alloc] initWithFD:fd];
+    FDInputStream *fdis = [[FDInputStream alloc] initWithFD:fd label:localPath];
     BufferedInputStream *bis = [[BufferedInputStream alloc] initWithUnderlyingStream:fdis];
     do {
         if (lseek(fd, offset, SEEK_SET) == -1) {
-            SETNSERROR(@"UnixErrorDomain", errno, @"lseek(%@, %qu): %s", localPath, offset, strerror(errno));
+            int errnum = errno;
+            HSLogError(@"lseek(%@, %qu) error %d: %s", localPath, offset, errnum, strerror(errnum));
+            SETNSERROR(@"UnixErrorDomain", errnum, @"failed to seek to %qu in %@: %s", offset, localPath, strerror(errnum));
             break;
         }
         NSString *mimeType;
@@ -131,10 +138,12 @@
         }
         NSData *data = nil;
         if (dataLen > 0) {
-            data = [bis readExactly:dataLen error:error];
+            unsigned char *buf = (unsigned char *)malloc(dataLen);
-            if (data == nil) {
+            if (![bis readExactly:dataLen into:buf error:error]) {
+                free(buf);
                 break;
             }
+            data = [NSData dataWithBytesNoCopy:buf length:dataLen];
         } else {
             data = [NSData data];
         }
@@ -148,12 +157,32 @@
 - (BOOL)fileLength:(unsigned long long *)length error:(NSError **)error {
     struct stat st;
     if (lstat([localPath fileSystemRepresentation], &st) == -1) {
-        SETNSERROR(@"UnixErrorDomain", errno, @"lstat(%@): %s", localPath, strerror(errno));
+        int errnum = errno;
+        HSLogError(@"lstat(%@) error %d: %s", localPath, errnum, strerror(errnum));
+        SETNSERROR(@"UnixErrorDomain", errnum, @"%@: %s", localPath, strerror(errnum));
         return NO;
     }
     *length = st.st_size;
     return YES;
 }
+- (BOOL)copyToPath:(NSString *)dest error:(NSError **)error {
+    NSFileManager *fm = [NSFileManager defaultManager];
+    if ([fm fileExistsAtPath:dest] && ![fm removeItemAtPath:dest error:error]) {
+        HSLogError(@"error removing old mutable pack at %@", dest);
+        return NO;
+    }
+    if (![fm ensureParentPathExistsForPath:dest targetUID:targetUID targetGID:targetGID error:error] || ![fm copyItemAtPath:localPath toPath:dest error:error]) {
+        HSLogError(@"error copying pack %@ to %@", localPath, dest);
+        return NO;
+    }
+    if (chown([localPath fileSystemRepresentation], targetUID, targetGID) == -1) {
+        int errnum = errno;
+        SETNSERROR(@"UnixErrorDomain", errnum, @"chown(%@): %s", localPath, strerror(errnum));
+        return NO;
+    }
+    HSLogDebug(@"copied %@ to %@", localPath, dest);
+    return YES;
+}
 - (NSArray *)sortedPackIndexEntries:(NSError **)error {
     unsigned long long length;
     if (![self fileLength:&length error:error]) {
@@ -169,15 +198,16 @@
 @end
 
 @implementation  DiskPack (internal)
-- (BOOL)savePack:(ServerBlob *)sb bytesWritten:(unsigned long long *)written error:(NSError **)error {
+- (BOOL)savePack:(ServerBlob *)sb error:(NSError **)error {
-    if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:localPath error:error]) {
+    if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:localPath targetUID:targetUID targetGID:targetGID error:error]) {
         return NO;
     }
     id <InputStream> is = [sb newInputStream];
     NSError *myError;
-    BOOL ret = [Streams transferFrom:is atomicallyToFile:localPath bytesWritten:written error:&myError];
+    unsigned long long written;
+    BOOL ret = [Streams transferFrom:is atomicallyToFile:localPath targetUID:targetUID targetGID:targetGID bytesWritten:&written error:&myError];
     if (ret) {
-        HSLogDebug(@"wrote %qu bytes to %@", *written, localPath);
+        HSLogDebug(@"wrote %qu bytes to %@", written, localPath);
     } else {
         if (error != NULL) {
             *error = myError;
@@ -224,4 +254,9 @@
     }
     return ret;    
 }
+
+#pragma mark NSObject
+- (NSString *)description {
+    return [NSString stringWithFormat:@"<DiskPack s3Bucket=%@ computerUUID=%@ packset=%@ sha1=%@ localPath=%@>", s3BucketName, computerUUID, packSetName, packSHA1, localPath];
+}
 @end

DiskPackIndex.h

@@ -1,34 +1,10 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  DiskPackIndex.h
- 
+//  Arq
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 12/30/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 __MyCompanyName__. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #import <Cocoa/Cocoa.h>
 @class PackIndexEntry;
@@ -42,11 +18,29 @@
     NSString *packSHA1;
     NSString *s3Path;
     NSString *localPath;
+    uid_t targetUID;
+    gid_t targetGID;
 }
 + (NSString *)s3PathWithS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1;
-+ (NSString *)localPathWithComputerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1;
++ (NSString *)localPathWithS3BucketName:theS3BucketName computerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1;
-- (id)initWithS3Service:(S3Service *)theS3 s3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1;
++ (NSArray *)diskPackIndexesForS3Service:(S3Service *)theS3
+                            s3BucketName:theS3BucketName 
+                            computerUUID:(NSString *)theComputerUUID 
+                             packSetName:(NSString *)thePackSetName 
+                               targetUID:(uid_t)theTargetUID 
+                               targetGID:(gid_t)theTargetGID
+                                   error:(NSError **)error;
+
+- (id)initWithS3Service:(S3Service *)theS3 
+           s3BucketName:(NSString *)theS3BucketName 
+           computerUUID:(NSString *)theComputerUUID 
+            packSetName:(NSString *)thePackSetName 
+               packSHA1:(NSString *)thePackSHA1
+              targetUID:(uid_t)theTargetUID 
+              targetGID:(gid_t)theTargetGID;
 - (BOOL)makeLocal:(NSError **)error;
 - (NSArray *)allPackIndexEntries:(NSError **)error;
 - (PackIndexEntry *)entryForSHA1:(NSString *)sha1 error:(NSError **)error;
+- (NSString *)packSetName;
+- (NSString *)packSHA1;
 @end

DiskPackIndex.m

@@ -1,34 +1,10 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  DiskPackIndex.m
- 
+//  Arq
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 12/30/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 __MyCompanyName__. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #include <sys/stat.h>
 #include <sys/mman.h>
@@ -49,7 +25,9 @@
 #import "BlobACL.h"
 #import "FileInputStreamFactory.h"
 #import "PackIndexWriter.h"
-#import "ArqUserLibrary.h"
+#import "UserLibrary_Arq.h"
+#import "NSError_extra.h"
+#import "RegexKitLite.h"
 
 typedef struct index_object {
     uint64_t nbo_offset;
@@ -66,7 +44,7 @@ typedef struct pack_index {
 } pack_index;
 
 @interface DiskPackIndex (internal)
-- (BOOL)savePackIndex:(ServerBlob *)sb bytesWritten:(unsigned long long *)written error:(NSError **)error;
+- (BOOL)savePackIndex:(ServerBlob *)sb error:(NSError **)error;
 - (PackIndexEntry *)doEntryForSHA1:(NSString *)sha1 error:(NSError **)error;
 - (PackIndexEntry *)findEntryForSHA1:(NSString *)sha1 fd:(int)fd betweenStartIndex:(uint32_t)startIndex andEndIndex:(uint32_t)endIndex error:(NSError **)error;
 - (BOOL)readFanoutStartIndex:(uint32_t *)start fanoutEndIndex:(uint32_t *)end fromFD:(int)fd forSHA1FirstByte:(unsigned int)firstByte error:(NSError **)error;
@@ -76,10 +54,48 @@ typedef struct pack_index {
 + (NSString *)s3PathWithS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1 {
     return [NSString stringWithFormat:@"/%@/%@/packsets/%@/%@.index", theS3BucketName, theComputerUUID, thePackSetName, thePackSHA1];
 }
-+ (NSString *)localPathWithComputerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1 {
++ (NSString *)localPathWithS3BucketName:theS3BucketName computerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1 {
-    return [NSString stringWithFormat:@"%@/%@/packsets/%@/%@/%@.index", [ArqUserLibrary arqCachesPath], theComputerUUID, thePackSetName, [thePackSHA1 substringToIndex:2], [thePackSHA1 substringFromIndex:2]];
+    return [NSString stringWithFormat:@"%@/%@/%@/packsets/%@/%@/%@.index", [UserLibrary arqCachePath], theS3BucketName, theComputerUUID, thePackSetName, [thePackSHA1 substringToIndex:2], [thePackSHA1 substringFromIndex:2]];
 }
-- (id)initWithS3Service:(S3Service *)theS3 s3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID packSetName:(NSString *)thePackSetName packSHA1:(NSString *)thePackSHA1 {
++ (NSArray *)diskPackIndexesForS3Service:(S3Service *)theS3
+                            s3BucketName:theS3BucketName 
+                            computerUUID:(NSString *)theComputerUUID 
+                             packSetName:(NSString *)thePackSetName 
+                               targetUID:(uid_t)theTargetUID 
+                               targetGID:(gid_t)theTargetGID
+                                   error:(NSError **)error {
+    NSMutableArray *diskPackIndexes = [NSMutableArray array];
+    NSString *packSetsPrefix = [NSString stringWithFormat:@"/%@/%@/packsets/%@/", theS3BucketName, theComputerUUID, thePackSetName];
+    NSArray *paths = [theS3 pathsWithPrefix:packSetsPrefix error:error];
+    if (paths == nil) {
+        return nil;
+    }
+    for (NSString *thePath in paths) {
+        NSRange sha1Range = [thePath rangeOfRegex:@"/(\\w+)\\.pack$" capture:1];
+        if (sha1Range.location != NSNotFound) {
+            NSString *thePackSHA1 = [thePath substringWithRange:sha1Range];
+            DiskPackIndex *index = [[DiskPackIndex alloc] initWithS3Service:theS3 
+                                                               s3BucketName:theS3BucketName
+                                                               computerUUID:theComputerUUID 
+                                                                packSetName:thePackSetName 
+                                                                   packSHA1:thePackSHA1 
+                                                                  targetUID:theTargetUID 
+                                                                  targetGID:theTargetGID];
+            [diskPackIndexes addObject:index];
+            [index release];
+        }            
+    }
+    return diskPackIndexes;
+}
+
+
+- (id)initWithS3Service:(S3Service *)theS3 
+           s3BucketName:(NSString *)theS3BucketName 
+           computerUUID:(NSString *)theComputerUUID 
+            packSetName:(NSString *)thePackSetName 
+               packSHA1:(NSString *)thePackSHA1
+              targetUID:(uid_t)theTargetUID 
+              targetGID:(gid_t)theTargetGID {
     if (self = [super init]) {
         s3 = [theS3 retain];
         s3BucketName = [theS3BucketName retain];
@@ -87,7 +103,9 @@ typedef struct pack_index {
         packSetName = [thePackSetName retain];
         packSHA1 = [thePackSHA1 retain];
         s3Path = [[DiskPackIndex s3PathWithS3BucketName:s3BucketName computerUUID:computerUUID packSetName:packSetName packSHA1:packSHA1] retain];
-        localPath = [[DiskPackIndex localPathWithComputerUUID:computerUUID packSetName:packSetName packSHA1:packSHA1] retain];
+        localPath = [[DiskPackIndex localPathWithS3BucketName:s3BucketName computerUUID:computerUUID packSetName:packSetName packSHA1:packSHA1] retain];
+        targetUID = theTargetUID;
+        targetGID = theTargetGID;
     }
     return self;
 }
@@ -103,37 +121,55 @@ typedef struct pack_index {
 }
 - (BOOL)makeLocal:(NSError **)error {
     NSFileManager *fm = [NSFileManager defaultManager];
-    BOOL ret = NO;
+    BOOL ret = YES;
     if (![fm fileExistsAtPath:localPath]) {
-        HSLogDebug(@"packset %@: making pack index %@ local", packSetName, packSHA1);
+        for (;;) {
-        ServerBlob *sb = [s3 newServerBlobAtPath:s3Path error:error];
+            HSLogDebug(@"packset %@: making pack index %@ local", packSetName, packSHA1);
-        if (sb == nil) {
+            NSError *myError = nil;
-            ret = NO;
+            ServerBlob *sb = [s3 newServerBlobAtPath:s3Path error:&myError];
-        } else {
+            if (sb != nil) {
-            unsigned long long bytesWritten;
+                ret = [self savePackIndex:sb error:error];
-            ret = [self savePackIndex:sb bytesWritten:&bytesWritten error:error];
+                [sb release];
-            [sb release];
+                break;
+            }
+            if (![myError isTransientError]) {
+                HSLogError(@"error getting S3 pack index %@: %@", s3Path, myError);
+                if (error != NULL) {
+                    *error = myError;
+                }
+                ret = NO;
+                break;
+            }
+            HSLogWarn(@"network error making pack index %@ local (retrying): %@", s3Path, myError);
+            NSError *rmError = nil;
+            if (![[NSFileManager defaultManager] removeItemAtPath:localPath error:&rmError]) {
+                HSLogError(@"error deleting incomplete downloaded pack index %@: %@", localPath, rmError);
+            }
         }
-    } else {
-        ret = YES;
     }
     return ret;
 }
 - (NSArray *)allPackIndexEntries:(NSError **)error {
     int fd = open([localPath fileSystemRepresentation], O_RDONLY);
     if (fd == -1) {
-        SETNSERROR(@"UnixErrorDomain", errno, @"%s: %@", strerror(errno), localPath);
+        int errnum = errno;
+        HSLogError(@"open(%@) error %d: %s", localPath, errnum, strerror(errnum));
+        SETNSERROR(@"UnixErrorDomain", errnum, @"failed to open %@: %s", localPath, strerror(errnum));
         return nil;
     }
     struct stat st;
     if (fstat(fd, &st) == -1) {
-        SETNSERROR(@"UnixErrorDomain", errno, @"fstat(%@): %s", localPath, strerror(errno));
+        int errnum = errno;
+        HSLogError(@"fstat(%@) error %d: %s", localPath, errnum, strerror(errnum));
+        SETNSERROR(@"UnixErrorDomain", errnum, @"%@: %s", localPath, strerror(errnum));
         close(fd);
         return nil;
     }
     pack_index *the_pack_index = mmap(0, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
     if (the_pack_index == MAP_FAILED) {
-        SETNSERROR(@"UnixErrorDomain", errno, @"mmap: %s", strerror(errno));
+        int errnum = errno;
+        HSLogError(@"mmap(%@) error %d: %s", localPath, errnum, strerror(errnum));
+        SETNSERROR(@"UnixErrorDomain", errnum, @"error mapping %@ to memory: %s", localPath, strerror(errnum));
         close(fd);
         return NO;
     }
@@ -150,7 +186,8 @@ typedef struct pack_index {
         [pool drain];
     }
     if (munmap(the_pack_index, st.st_size) == -1) {
-        HSLogError(@"munmap: %s", strerror(errno));
+        int errnum = errno;
+        HSLogError(@"munmap: %s", strerror(errnum));
     }
     close(fd);
     return ret;
@@ -172,18 +209,30 @@ typedef struct pack_index {
     }
     return ret;
 }
+- (NSString *)packSetName {
+    return packSetName;
+}
+- (NSString *)packSHA1 {
+    return packSHA1;
+}
+
+#pragma mark NSObject
+- (NSString *)description {
+    return [NSString stringWithFormat:@"<DiskPackIndex: computerUUID=%@ packset=%@ packSHA1=%@>", computerUUID, packSetName, packSHA1];
+}
 @end
 
 @implementation DiskPackIndex (internal)
-- (BOOL)savePackIndex:(ServerBlob *)sb bytesWritten:(unsigned long long *)written error:(NSError **)error {
+- (BOOL)savePackIndex:(ServerBlob *)sb error:(NSError **)error {
-    if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:localPath error:error]) {
+    if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:localPath targetUID:targetUID targetGID:targetGID error:error]) {
         return NO;
     }
     id <InputStream> is = [sb newInputStream];
     NSError *myError = nil;
-    BOOL ret = [Streams transferFrom:is atomicallyToFile:localPath bytesWritten:written error:&myError];
+    unsigned long long written = 0;
+    BOOL ret = [Streams transferFrom:is atomicallyToFile:localPath targetUID:targetUID targetGID:targetGID bytesWritten:&written error:&myError];
     if (ret) {
-        HSLogDebug(@"wrote %qu bytes to %@", *written, localPath);
+        HSLogDebug(@"wrote %qu bytes to %@", written, localPath);
     } else {
         if (error != NULL) {
             *error = myError;
@@ -199,7 +248,9 @@ typedef struct pack_index {
     HSLogTrace(@"looking for sha1 %@ in packindex %@", sha1, packSHA1);
     int fd = open([localPath fileSystemRepresentation], O_RDONLY);
     if (fd == -1) {
-        SETNSERROR(@"UnixErrorDomain", errno, @"%s: %@", strerror(errno), localPath);
+        int errnum = errno;
+        HSLogError(@"open(%@) error %d: %s", localPath, errnum, strerror(errnum));
+        SETNSERROR(@"UnixErrorDomain", errnum, @"failed to open %@: %s", localPath, strerror(errnum));
         return nil;
     }
     uint32_t startIndex;
@@ -215,7 +266,9 @@ typedef struct pack_index {
     }
     fd = open([localPath fileSystemRepresentation], O_RDONLY);
     if (fd == -1) {
-        SETNSERROR(@"UnixErrorDomain", errno, @"%s: %@", strerror(errno), localPath);
+        int errnum = errno;
+        HSLogError(@"open(%@) error %d: %s", localPath, errnum, strerror(errnum));
+        SETNSERROR(@"UnixErrorDomain", errnum, @"failed to open %@: %s", localPath, strerror(errnum));
         return nil;
     }
     PackIndexEntry *ret = [self findEntryForSHA1:sha1 fd:fd betweenStartIndex:startIndex andEndIndex:endIndex error:error];
@@ -231,7 +284,9 @@ typedef struct pack_index {
     uint32_t lengthToMap = 4 + 4 + 256*4 + endIndex * sizeof(index_object);
     pack_index *the_pack_index = mmap(0, lengthToMap, PROT_READ, MAP_SHARED, fd, 0);
     if (the_pack_index == MAP_FAILED) {
-        SETNSERROR(@"UnixErrorDomain", errno, @"mmap: %s", strerror(errno));
+        int errnum = errno;
+        HSLogError(@"mmap(%@) error %d: %s", localPath, errnum, strerror(errnum));
+        SETNSERROR(@"UnixErrorDomain", errnum, @"error mapping %@ to memory: %s", localPath, strerror(errnum));
         return NO;
     }
     int64_t left = startIndex;
@@ -263,7 +318,8 @@ typedef struct pack_index {
         }
     }
     if (munmap(the_pack_index, lengthToMap) == -1) {
-        HSLogError(@"munmap: %s", strerror(errno));
+        int errnum = errno;
+        HSLogError(@"munmap: %s", strerror(errnum));
     }
     if (pie == nil) {
         SETNSERROR(@"PackErrorDomain", ERROR_NOT_FOUND, @"sha1 %@ not found in pack %@", sha1, packSHA1);
@@ -274,7 +330,9 @@ typedef struct pack_index {
     size_t len = 4 + 4 + 4*256;
     uint32_t *map = mmap(0, len, PROT_READ, MAP_SHARED, fd, 0);
     if (map == MAP_FAILED) {
-        SETNSERROR(@"UnixErrorDomain", errno, @"mmap: %s", strerror(errno));
+        int errnum = errno;
+        HSLogError(@"mmap(%@) error %d: %s", localPath, errnum, strerror(errnum));
+        SETNSERROR(@"UnixErrorDomain", errnum, @"error mapping %@ to memory: %s", localPath, strerror(errnum));
         return NO;
     }
     BOOL ret = YES;
@@ -292,7 +350,8 @@ typedef struct pack_index {
         *end = OSSwapBigToHostInt32(fanoutTable[firstByte]);
     }
     if (munmap(map, len) == -1) {
-        HSLogError(@"munmap: %s", strerror(errno));
+        int errnum = errno;
+        HSLogError(@"munmap: %s", strerror(errnum));
     }
     return ret;
 }

FarkPath.h

@@ -13,5 +13,5 @@
 
 }
 + (NSString *)s3PathForS3BucketName:(NSString *)s3BucketName computerUUID:(NSString *)computerUUID sha1:(NSString *)sha1;
-+ (NSString *)s3PathForBucketDataPath:(NSString *)bucketDataPath s3BucketName:(NSString *)s3BucketName computerUUID:(NSString *)computerUUID;
++ (NSString *)s3PathForBucketDataRelativePath:(NSString *)bucketDataRelativePath s3BucketName:(NSString *)s3BucketName computerUUID:(NSString *)computerUUID;
 @end

FarkPath.m

@@ -13,7 +13,7 @@
 + (NSString *)s3PathForS3BucketName:(NSString *)s3BucketName computerUUID:(NSString *)computerUUID sha1:(NSString *)sha1 {
     return [NSString stringWithFormat:@"/%@/%@/objects/%@", s3BucketName, computerUUID, sha1];
 }
-+ (NSString *)s3PathForBucketDataPath:(NSString *)bucketDataPath s3BucketName:(NSString *)s3BucketName computerUUID:(NSString *)computerUUID {
++ (NSString *)s3PathForBucketDataRelativePath:(NSString *)bucketDataRelativePath s3BucketName:(NSString *)s3BucketName computerUUID:(NSString *)computerUUID {
-    return [[NSString stringWithFormat:@"/%@/%@/bucketdata", s3BucketName, computerUUID] stringByAppendingPathComponent:bucketDataPath];
+    return [[NSString stringWithFormat:@"/%@/%@/bucketdata", s3BucketName, computerUUID] stringByAppendingPathComponent:bucketDataRelativePath];
 }
 @end

FileAttributes.h

@@ -1,34 +1,10 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  FileAttributes.h
- 
+//  Backup
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 4/22/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 PhotoMinds LLC. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #include <sys/stat.h>
 #import <Cocoa/Cocoa.h>
@@ -39,7 +15,6 @@
     const char *cPath;
     struct stat st;
     struct timespec createTime;
-    NSString *aclString;
     int finderFlags;
     int extendedFinderFlags;
     NSString *finderFileType;
@@ -49,8 +24,6 @@
 - (id)initWithPath:(NSString *)thePath stat:(struct stat *)st error:(NSError **)error;
 
 - (unsigned long long)fileSize;
-- (NSString *)aclString;
-- (NSString *)aclSHA1;
 - (int)uid;
 - (int)gid;
 - (int)mode;
@@ -80,7 +53,6 @@
 
 - (BOOL)applyFinderFileType:(NSString *)finderFileType finderFileCreator:(NSString *)finderFileCreator error:(NSError **)error;
 - (BOOL)applyFlags:(int)flags error:(NSError **)error;
-- (BOOL)applyAcl:(NSString *)aclString error:(NSError **)error;
 - (BOOL)applyFinderFlags:(int)finderFlags error:(NSError **)error;
 - (BOOL)applyExtendedFinderFlags:(int)extendedFinderFlags error:(NSError **)error;
 - (BOOL)applyExtensionHidden:(BOOL)isExtensionHidden error:(NSError **)error;

FileAttributes.m

@@ -1,34 +1,10 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  FileAttributes.m
- 
+//  Backup
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 4/22/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 PhotoMinds LLC. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #include <CoreServices/CoreServices.h>
 #include <sys/attr.h>
@@ -40,6 +16,7 @@
 #import "FileACL.h"
 #import "SetNSError.h"
 #import "OSStatusDescription.h"
+#import "NSError_extra.h"
 
 #define kCouldNotCreateCFString	4
 #define kCouldNotGetStringData	5
@@ -103,7 +80,9 @@ static OSStatus SymlinkPathMakeRef(const UInt8 *path, FSRef *ref, Boolean *isDir
     struct stat theStat;
     int ret = lstat([thePath fileSystemRepresentation], &theStat);
     if (ret == -1) {
-        SETNSERROR(@"UnixErrorDomain", errno, @"%s", strerror(errno));
+        int errnum = errno;
+        HSLogError(@"lstat(%@) error %d: %s", thePath, errnum, strerror(errnum));
+        SETNSERROR(@"UnixErrorDomain", errnum, @"%@: %s", thePath, strerror(errnum));
         return nil;
     }
     return [self initWithPath:thePath stat:&theStat error:error];
@@ -133,7 +112,7 @@ static OSStatus SymlinkPathMakeRef(const UInt8 *path, FSRef *ref, Boolean *isDir
             if (oss == bdNamErr) {
                 HSLogInfo(@"skipping finder flags for %s: %@", cPath, [OSStatusDescription descriptionForMacOSStatus:oss]);
             }else if (oss != noErr) {
-                SETNSERROR(@"MacFilesErrorDomain", oss, @"%@", [OSStatusDescription descriptionForMacOSStatus:oss]);
+                SETNSERROR(@"MacFilesErrorDomain", oss, @"error making FSRef for %@: %@", thePath, [OSStatusDescription descriptionForMacOSStatus:oss]);
                 [self release];
                 self = nil;
                 return self;
@@ -141,11 +120,21 @@ static OSStatus SymlinkPathMakeRef(const UInt8 *path, FSRef *ref, Boolean *isDir
                 FSCatalogInfo catalogInfo;
                 OSErr oserr = FSGetCatalogInfo(&fsRef, kFSCatInfoCreateDate | kFSCatInfoFinderInfo | kFSCatInfoFinderXInfo, &catalogInfo, NULL, NULL, NULL);
                 if (oserr) {
-                    SETNSERROR(@"MacFilesErrorDomain", oss, @"%@", [OSStatusDescription descriptionForMacOSStatus:(OSStatus)oserr]);
+                    SETNSERROR(@"MacFilesErrorDomain", oss, @"FSGetCatalogInfo(%@): %@", thePath, [OSStatusDescription descriptionForMacOSStatus:(OSStatus)oserr]);
                     [self release];
                     self = nil;
                     return self;
                 }
+                
+                CFTimeInterval theCreateTime; // double: seconds since reference date
+                if (UCConvertUTCDateTimeToCFAbsoluteTime(&catalogInfo.createDate, &theCreateTime) != noErr) {
+                    HSLogError(@"error converting create time %f to CFAbsoluteTime", catalogInfo.createDate);
+                } else {
+                    createTime.tv_sec = (int64_t)(theCreateTime + NSTimeIntervalSince1970);
+                    CFTimeInterval subsecond = theCreateTime - (double)((int64_t)theCreateTime);
+                    createTime.tv_nsec = (int64_t)(subsecond * 1000000000.0);
+                }
+                
                 finderFlags = 0;
                 extendedFinderFlags = 0;
                 if (isDirectory) {
@@ -178,33 +167,11 @@ static OSStatus SymlinkPathMakeRef(const UInt8 *path, FSRef *ref, Boolean *isDir
                 }
             }
         }
-        
-        if (![FileACL aclText:&aclString forFile:path error:error]) {
-            [self release];
-            self = nil;
-            return self;
-        }
-        [aclString retain];
-        
-        struct attrlist attrList;
-        memset(&attrList, 0, sizeof(attrList));
-        attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
-        attrList.commonattr = ATTR_CMN_CRTIME;
-        struct createDateBuf createDateBuf;
-        if (getattrlist(cPath, &attrList, &createDateBuf, sizeof(createDateBuf), FSOPT_NOFOLLOW) == -1) {
-            SETNSERROR(@"UnixErrorDomain", errno, @"Error getting create date for %@: %s", thePath, strerror(errno));
-            [self release];
-            self = nil;
-            return self;
-        }
-        createTime.tv_sec = createDateBuf.createTime.tv_sec;
-        createTime.tv_nsec = createDateBuf.createTime.tv_nsec;
     }
     return self;
 }
 - (void)dealloc {
     [path release];
-    [aclString release];
     [finderFileType release];
     [finderFileCreator release];
     [super dealloc];
@@ -212,16 +179,6 @@ static OSStatus SymlinkPathMakeRef(const UInt8 *path, FSRef *ref, Boolean *isDir
 - (unsigned long long)fileSize {
     return (unsigned long long)st.st_size;
 }
-- (NSString *)aclString {
-    return aclString;
-}
-- (NSString *)aclSHA1 {
-    NSString *sha1 = nil;
-    if (aclString) {
-        sha1 = [SHA1Hash hashData:[aclString dataUsingEncoding:NSUTF8StringEncoding]];
-    }
-    return sha1;
-}
 - (int)uid {
     return st.st_uid;
 }
@@ -368,24 +325,15 @@ static OSStatus SymlinkPathMakeRef(const UInt8 *path, FSRef *ref, Boolean *isDir
     if (targetExists && flags != st.st_flags) {
         HSLogTrace(@"chflags(%s, %d)", cPath, flags);
         if (chflags(cPath, flags) == -1) {
-            SETNSERROR(@"UnixErrorDomain", errno, @"chflags: %s", strerror(errno));
+            int errnum = errno;
+            HSLogError(@"chflags(%s, %d) error %d: %s", cPath, flags, errnum, strerror(errnum));
+            SETNSERROR(@"UnixErrorDomain", errnum, @"error changing flags of %s: %s", cPath, strerror(errnum));
             return NO;
         }
         st.st_flags = flags;
     }
     return YES;
 }
-- (BOOL)applyAcl:(NSString *)theACLString error:(NSError **)error {
-    BOOL ret = YES;
-    if (![theACLString isEqualToString:aclString]) {
-        ret = [FileACL writeACLText:theACLString toFile:path error:error];
-        if (ret) {
-            [aclString release];
-            aclString = [theACLString retain];
-        }
-    }
-    return ret;
-}
 - (BOOL)applyFinderFlags:(int)ff error:(NSError **)error {
     if (targetExists && ff != finderFlags) {
         FSRef fsRef;
@@ -486,8 +434,9 @@ static OSStatus SymlinkPathMakeRef(const UInt8 *path, FSRef *ref, Boolean *isDir
 - (BOOL)applyUID:(int)uid gid:(int)gid error:(NSError **)error {
     if (uid != st.st_uid || gid != st.st_gid) {
         if (lchown(cPath, uid, gid) == -1) {
-            HSLogError(@"lchown failed");
+            int errnum = errno;
-            SETNSERROR(@"UnixErrorDomain", errno, @"lchown: %s", strerror(errno));
+            HSLogError(@"lchown(%s) error %d: %s", cPath, errnum, strerror(errnum));
+            SETNSERROR(@"UnixErrorDomain", errnum, @"error changing ownership of %s: %s", cPath, strerror(errnum));
             return NO;
         }
         HSLogDebug(@"lchown(%s, %d, %d); euid=%d", cPath, uid, gid, geteuid());
@@ -501,20 +450,26 @@ static OSStatus SymlinkPathMakeRef(const UInt8 *path, FSRef *ref, Boolean *isDir
         if (S_ISDIR(st.st_mode)) {
             int ret = chmod(cPath, mode);
             if (ret == -1) {
-                SETNSERROR(@"UnixErrorDomain", errno, @"Setting permissions on %@: %s", path, strerror(errno));
+                int errnum = errno;
+                HSLogError(@"chmod(%s, %d) error %d: %s", cPath, mode, errnum, strerror(errnum));
+                SETNSERROR(@"UnixErrorDomain", errnum, @"failed to set permissions on %@: %s", path, strerror(errnum));
                 return NO;
             }
             HSLogDebug(@"chmod(%s, 0%6o)", cPath, mode);
         } else {
             int fd = open(cPath, O_RDWR|O_SYMLINK);
             if (fd == -1) {
-                SETNSERROR(@"UnixErrorDomain", errno, @"%s", strerror(errno));
+                int errnum = errno;
+                HSLogError(@"open(%s) error %d: %s", cPath, errnum, strerror(errnum));
+                SETNSERROR(@"UnixErrorDomain", errnum, @"failed to open %@: %s", path, strerror(errnum));
                 return NO;
             }
             int ret = fchmod(fd, mode);
             close(fd);
             if (ret == -1) {
-                SETNSERROR(@"UnixErrorDomain", errno, @"Setting permissions on %@: %s", path, strerror(errno));
+                int errnum = errno;
+                HSLogError(@"fchmod(%@) error %d: %s", path, errnum, strerror(errnum));
+                SETNSERROR(@"UnixErrorDomain", errnum, @"failed to set permissions on %@: %s", path, strerror(errnum));
                 return NO;
             }
             HSLogDebug(@"fchmod(%s, 0%6o)", cPath, mode);
@@ -535,7 +490,9 @@ static OSStatus SymlinkPathMakeRef(const UInt8 *path, FSRef *ref, Boolean *isDir
         timevals[0] = atimeVal;
         timevals[1] = mtimeVal;
         if (utimes(cPath, timevals) == -1) {
-            SETNSERROR(@"UnixErrorDomain", errno, @"utimes(%@): %s", path, strerror(errno));
+            int errnum = errno;
+            HSLogError(@"utimes(%@) error %d: %s", path, errnum, strerror(errnum));
+            SETNSERROR(@"UnixErrorDomain", errnum, @"failed to set timestamps on %@: %s", path, strerror(errnum));
             return NO;
         }
     }
@@ -543,17 +500,41 @@ static OSStatus SymlinkPathMakeRef(const UInt8 *path, FSRef *ref, Boolean *isDir
 }
 - (BOOL)applyCreateTimeSec:(int64_t)theCreateTime_sec createTimeNSec:(int64_t)theCreateTime_nsec error:(NSError **)error {
     if (createTime.tv_sec != theCreateTime_sec || createTime.tv_nsec != theCreateTime_nsec) {
-        createTime.tv_sec = theCreateTime_sec;
+        FSRef fsRef;
-        createTime.tv_nsec = theCreateTime_nsec;
+        Boolean isDirectory;
-        
+        OSStatus oss = 0;
-        struct attrlist attrList;
+        if (S_ISLNK(st.st_mode)) {
-        memset(&attrList, 0, sizeof(attrList));
+            oss = SymlinkPathMakeRef((UInt8*)cPath, &fsRef, &isDirectory);
-        attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
+        } else {
-        attrList.commonattr = ATTR_CMN_CRTIME;
+            oss = FSPathMakeRef((UInt8*)cPath, &fsRef, &isDirectory);
-        if (setattrlist(cPath, &attrList, &createTime, sizeof(createTime), FSOPT_NOFOLLOW) == -1) {
+        }
-            SETNSERROR(@"UnixErrorDomain", errno, @"Error setting create date on %@: %s", path, strerror(errno));
+        if (oss != noErr) {
+            if (oss == bdNamErr) {
+                HSLogInfo(@"not setting create time on %s: bad name", cPath);
+                return YES;
+            } else {
+                SETNSERROR(@"FileManagerErrorDomain", -1, @"%@", [OSStatusDescription descriptionForMacOSStatus:oss]);
+                return NO;
+            }
+        }
+        FSCatalogInfo catalogInfo;
+        OSErr oserr = FSGetCatalogInfo(&fsRef, kFSCatInfoCreateDate, &catalogInfo, NULL, NULL, NULL);
+        if (oserr) {
+            SETNSERROR(@"FileManagerErrorDomain", -1, @"%@", [OSStatusDescription descriptionForMacOSStatus:(OSStatus)oserr]);
             return NO;
         }
+        CFTimeInterval theCreateTime = (double)theCreateTime_sec - NSTimeIntervalSince1970 + (double)theCreateTime_nsec / 1000000000.0;
+        if (UCConvertCFAbsoluteTimeToUTCDateTime(theCreateTime, &catalogInfo.createDate) != noErr) {
+            SETNSERROR(@"FileManagerErrorDomain", -1, @"unable to convert CFAbsoluteTime %f to UTCDateTime", theCreateTime);
+            return NO;
+        }
+        oserr = FSSetCatalogInfo(&fsRef, kFSCatInfoCreateDate, &catalogInfo);
+        if (oserr) {
+            SETNSERROR(@"FileManagerErrorDomain", -1, @"%@", [OSStatusDescription descriptionForMacOSStatus:(OSStatus)oserr]);
+            return NO;
+        }
+        createTime.tv_sec = theCreateTime_sec;
+        createTime.tv_nsec = theCreateTime_nsec;
     }
     return YES;
 }

Node.h

@@ -1,48 +1,28 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  Node.h
- 
+//  s3print
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 4/10/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 PhotoMinds LLC. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #import <Cocoa/Cocoa.h>
 @protocol InputStream;
+@class BlobKey;
 
 @interface Node : NSObject {
     int treeVersion;
 	BOOL isTree;
-    unsigned long long dataSize;
+    unsigned long long uncompressedDataSize;
-    NSMutableArray *dataSHA1s;
+    BOOL dataAreCompressed;
-    NSString *thumbnailSHA1;
+    NSMutableArray *dataBlobKeys;
-    NSString *previewSHA1;
+    BlobKey *thumbnailBlobKey;
-	NSString *xattrsSHA1;
+    BlobKey *previewBlobKey;
+    BOOL xattrsAreCompressed;
+    BlobKey *xattrsBlobKey;
     unsigned long long xattrsSize;
-	NSString *aclSHA1;
+    BOOL aclIsCompressed;
+    BlobKey *aclBlobKey;
     int uid;
     int gid;
 	int mode;
@@ -70,15 +50,18 @@
 - (BOOL)dataMatchesStatData:(struct stat *)st;
 
 @property(readonly) BOOL isTree;
-@property(readonly,copy) NSString *treeSHA1;
+@property(readonly,copy) BlobKey *treeBlobKey;
-@property(readonly,copy) NSArray *dataSHA1s;
+@property(readonly) BOOL dataAreCompressed;
+@property(readonly,copy) NSArray *dataBlobKeys;
 
-@property(readonly) unsigned long long dataSize;
+@property(readonly) unsigned long long uncompressedDataSize;
-@property(readonly,copy) NSString *thumbnailSHA1;
+@property(readonly,copy) BlobKey *thumbnailBlobKey;
-@property(readonly,copy) NSString *previewSHA1;
+@property(readonly,copy) BlobKey *previewBlobKey;
-@property(readonly,copy) NSString *xattrsSHA1;
+@property(readonly) BOOL xattrsAreCompressed;
+@property(readonly,copy) BlobKey *xattrsBlobKey;
 @property(readonly) unsigned long long xattrsSize;
-@property(readonly,copy) NSString *aclSHA1;
+@property(readonly) BOOL aclIsCompressed;
+@property(readonly,copy) BlobKey *aclBlobKey;
 @property(readonly) int uid;
 @property(readonly) int gid;
 @property(readonly) int mode;
@@ -90,6 +73,7 @@
 @property(readonly,copy) NSString *finderFileType;
 @property(readonly,copy) NSString *finderFileCreator;
 @property(readonly) BOOL isFileExtensionHidden;
+@property(readonly) int st_dev;
 @property(readonly) int treeVersion;
 @property(readonly) int st_rdev;
 @property(readonly) long long ctime_sec;
@@ -98,4 +82,7 @@
 @property(readonly) long long createTime_nsec;
 @property(readonly) uint32_t st_nlink;
 @property(readonly) int st_ino;
+@property(readonly) int64_t st_blocks;
+@property(readonly) uint32_t st_blksize;
+- (uint64_t)sizeOnDisk;
 @end

Node.m

@@ -1,34 +1,10 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  Node.m
- 
+//  s3print
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 4/10/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 PhotoMinds LLC. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #include <sys/stat.h>
 #import "Node.h"
@@ -36,106 +12,154 @@
 #import "IntegerIO.h"
 #import "StringIO.h"
 #import "BufferedInputStream.h"
+#import "BlobKey.h"
+#import "NSObject_extra.h"
 
 @implementation Node
-@synthesize isTree, dataSize, thumbnailSHA1, previewSHA1, xattrsSHA1, xattrsSize, aclSHA1, uid, gid, mode, mtime_sec, mtime_nsec, flags, finderFlags, extendedFinderFlags, finderFileType, finderFileCreator, isFileExtensionHidden, treeVersion, st_rdev;
+@synthesize isTree, uncompressedDataSize, thumbnailBlobKey, previewBlobKey, xattrsBlobKey, xattrsSize, aclBlobKey, uid, gid, mode, mtime_sec, mtime_nsec, flags, finderFlags, extendedFinderFlags, finderFileType, finderFileCreator, isFileExtensionHidden, st_dev, treeVersion, st_rdev;
-@synthesize ctime_sec, ctime_nsec, createTime_sec, createTime_nsec, st_nlink, st_ino;
+@synthesize ctime_sec, ctime_nsec, createTime_sec, createTime_nsec, st_nlink, st_ino, st_blocks, st_blksize;
-@dynamic treeSHA1, dataSHA1s;
+@dynamic treeBlobKey, dataBlobKeys;
+@synthesize dataAreCompressed, xattrsAreCompressed, aclIsCompressed;
 
 - (id)initWithInputStream:(BufferedInputStream *)is treeVersion:(int)theTreeVersion error:(NSError **)error {
     if (self = [super init]) {
         treeVersion = theTreeVersion;
-        dataSHA1s = [[NSMutableArray alloc] init];
+        dataBlobKeys = [[NSMutableArray alloc] init];
-        BOOL ret = NO;
+
-        do {
+        if (![BooleanIO read:&isTree from:is error:error]) {
-            if (![BooleanIO read:&isTree from:is error:error]) {
+            [self release];
-                break;
+            return nil;
+        }
+        
+        if (treeVersion >= 12) {
+            if (![BooleanIO read:&dataAreCompressed from:is error:error]
+                || ![BooleanIO read:&xattrsAreCompressed from:is error:error]
+                || ![BooleanIO read:&aclIsCompressed from:is error:error]) {
+                [self release];
+                return nil;
             }
-            int dataSHA1sCount;
+        }
-            if (![IntegerIO readInt32:&dataSHA1sCount from:is error:error]) {
+        
-                break;
+        int dataBlobKeysCount;
+        if (![IntegerIO readInt32:&dataBlobKeysCount from:is error:error]) {
+            [self release];
+            return nil;
+        }
+        for (int i = 0; i < dataBlobKeysCount; i++) {
+            NSString *dataSHA1;
+            BOOL stretchEncryptionKey = NO;
+            if (![StringIO read:&dataSHA1 from:is error:error]) {
+                [self release];
+                return nil;
+            }            
+            if (treeVersion >= 14 && ![BooleanIO read:&stretchEncryptionKey from:is error:error]) {
+                [self release];
+                return nil;
             }
-            for (int i = 0; i < dataSHA1sCount; i++) {
+            BlobKey *bk = [[BlobKey alloc] initWithSHA1:dataSHA1 stretchEncryptionKey:stretchEncryptionKey];
-                NSString *dataSHA1;
+            [dataBlobKeys addObject:bk];
-                if (![StringIO read:&dataSHA1 from:is error:error]) {
+            [bk release];
-                    break;
+        }
-                }
+        NSString *thumbnailSHA1 = nil;
-                [dataSHA1s addObject:dataSHA1];
+        BOOL thumbnailStretchedKey = NO;
-            }
+        NSString *previewSHA1 = nil;
-            ret = [IntegerIO readUInt64:&dataSize from:is error:error]
+        BOOL previewStretchedKey = NO;
-            && [StringIO read:&thumbnailSHA1 from:is error:error]
+        NSString *xattrsSHA1 = nil;
-            && [StringIO read:&previewSHA1 from:is error:error]
+        BOOL xattrsStretchedKey = NO;
-            && [StringIO read:&xattrsSHA1 from:is error:error]
+        NSString *aclSHA1 = nil;
-            && [IntegerIO readUInt64:&xattrsSize from:is error:error]
+        BOOL aclStretchedKey = NO;
-            && [StringIO read:&aclSHA1 from:is error:error]
+        BOOL ret = [IntegerIO readUInt64:&uncompressedDataSize from:is error:error]
-            && [IntegerIO readInt32:&uid from:is error:error]
+        && [StringIO read:&thumbnailSHA1 from:is error:error]
-            && [IntegerIO readInt32:&gid from:is error:error]
+        && (treeVersion < 14 || [BooleanIO read:&thumbnailStretchedKey from:is error:error])
-            && [IntegerIO readInt32:&mode from:is error:error]
+        && [StringIO read:&previewSHA1 from:is error:error]
-            && [IntegerIO readInt64:&mtime_sec from:is error:error]
+        && (treeVersion < 14 || [BooleanIO read:&previewStretchedKey from:is error:error])
-            && [IntegerIO readInt64:&mtime_nsec from:is error:error]
+        && [StringIO read:&xattrsSHA1 from:is error:error]
-            && [IntegerIO readInt64:&flags from:is error:error]
+        && (treeVersion < 14 || [BooleanIO read:&xattrsStretchedKey from:is error:error])
-            && [IntegerIO readInt32:&finderFlags from:is error:error]
+        && [IntegerIO readUInt64:&xattrsSize from:is error:error]
-            && [IntegerIO readInt32:&extendedFinderFlags from:is error:error]
+        && [StringIO read:&aclSHA1 from:is error:error]
-            && [StringIO read:&finderFileType from:is error:error]
+        && (treeVersion < 14 || [BooleanIO read:&aclStretchedKey from:is error:error])
-            && [StringIO read:&finderFileCreator from:is error:error]
+        && [IntegerIO readInt32:&uid from:is error:error]
-            && [BooleanIO read:&isFileExtensionHidden from:is error:error]
+        && [IntegerIO readInt32:&gid from:is error:error]
-            && [IntegerIO readInt32:&st_dev from:is error:error]
+        && [IntegerIO readInt32:&mode from:is error:error]
-            && [IntegerIO readInt32:&st_ino from:is error:error]
+        && [IntegerIO readInt64:&mtime_sec from:is error:error]
-            && [IntegerIO readUInt32:&st_nlink from:is error:error]
+        && [IntegerIO readInt64:&mtime_nsec from:is error:error]
-            && [IntegerIO readInt32:&st_rdev from:is error:error]
+        && [IntegerIO readInt64:&flags from:is error:error]
-            && [IntegerIO readInt64:&ctime_sec from:is error:error]
+        && [IntegerIO readInt32:&finderFlags from:is error:error]
-            && [IntegerIO readInt64:&ctime_nsec from:is error:error]
+        && [IntegerIO readInt32:&extendedFinderFlags from:is error:error]
-            && [IntegerIO readInt64:&createTime_sec from:is error:error]
+        && [StringIO read:&finderFileType from:is error:error]
-            && [IntegerIO readInt64:&createTime_nsec from:is error:error]
+        && [StringIO read:&finderFileCreator from:is error:error]
-            && [IntegerIO readInt64:&st_blocks from:is error:error]
+        && [BooleanIO read:&isFileExtensionHidden from:is error:error]
-            && [IntegerIO readUInt32:&st_blksize from:is error:error];
+        && [IntegerIO readInt32:&st_dev from:is error:error]
-            [thumbnailSHA1 retain];
+        && [IntegerIO readInt32:&st_ino from:is error:error]
-            [previewSHA1 retain];
+        && [IntegerIO readUInt32:&st_nlink from:is error:error]
-            [xattrsSHA1 retain];
+        && [IntegerIO readInt32:&st_rdev from:is error:error]
-            [aclSHA1 retain];
+        && [IntegerIO readInt64:&ctime_sec from:is error:error]
-            [finderFileType retain];
+        && [IntegerIO readInt64:&ctime_nsec from:is error:error]
-            [finderFileCreator retain];
+        && [IntegerIO readInt64:&createTime_sec from:is error:error]
-        } while(0);
+        && [IntegerIO readInt64:&createTime_nsec from:is error:error]
+        && [IntegerIO readInt64:&st_blocks from:is error:error]
+        && [IntegerIO readUInt32:&st_blksize from:is error:error];
+        [finderFileType retain];
+        [finderFileCreator retain];
         if (!ret) {
             [self release];
-            self = nil;
             return nil;
         }
+        if (thumbnailSHA1 != nil) {
+            thumbnailBlobKey = [[BlobKey alloc] initWithSHA1:thumbnailSHA1 stretchEncryptionKey:thumbnailStretchedKey];
+        }
+        if (previewSHA1 != nil) {
+            previewBlobKey = [[BlobKey alloc] initWithSHA1:previewSHA1 stretchEncryptionKey:previewStretchedKey];
+        }
+        if (xattrsSHA1 != nil) {
+            xattrsBlobKey = [[BlobKey alloc] initWithSHA1:xattrsSHA1 stretchEncryptionKey:xattrsStretchedKey];
+        }
+        if (aclSHA1 != nil) {
+            aclBlobKey = [[BlobKey alloc] initWithSHA1:aclSHA1 stretchEncryptionKey:aclStretchedKey];
+        }
     }
     return self;
 }
 - (void)dealloc {
-	[dataSHA1s release];
+	[dataBlobKeys release];
-    [thumbnailSHA1 release];
+    [thumbnailBlobKey release];
-    [previewSHA1 release];
+    [previewBlobKey release];
-	[xattrsSHA1 release];
+	[xattrsBlobKey release];
-	[aclSHA1 release];
+	[aclBlobKey release];
 	[finderFileType release];
 	[finderFileCreator release];
 	[super dealloc];
 }
-- (NSString *)treeSHA1 {
+- (BlobKey *)treeBlobKey {
     NSAssert(isTree, @"must be a Tree");
-    return [dataSHA1s objectAtIndex:0];
+    return [dataBlobKeys objectAtIndex:0];
 }
-- (NSArray *)dataSHA1s {
+- (NSArray *)dataBlobKeys {
-    return dataSHA1s;
+    return dataBlobKeys;
 }
 - (BOOL)dataMatchesStatData:(struct stat *)st {
-    return (st->st_mtimespec.tv_sec == mtime_sec && st->st_mtimespec.tv_nsec == mtime_nsec && st->st_size == dataSize);
+    return (st->st_mtimespec.tv_sec == mtime_sec && st->st_mtimespec.tv_nsec == mtime_nsec && st->st_size == uncompressedDataSize);
 }
 - (void)writeToData:(NSMutableData *)data {
     [BooleanIO write:isTree to:data];
-    [IntegerIO writeInt32:(int32_t)[dataSHA1s count] to:data];
+    [BooleanIO write:dataAreCompressed to:data];
-    for (NSString *dataSHA1 in dataSHA1s) {
+    [BooleanIO write:xattrsAreCompressed to:data];
-        [StringIO write:dataSHA1 to:data];
+    [BooleanIO write:aclIsCompressed to:data];
+    [IntegerIO writeInt32:(int32_t)[dataBlobKeys count] to:data];
+    for (BlobKey *dataBlobKey in dataBlobKeys) {
+        [StringIO write:[dataBlobKey sha1] to:data];
+        [BooleanIO write:[dataBlobKey stretchEncryptionKey] to:data];
     }
-    [IntegerIO writeUInt64:dataSize to:data];
+    [IntegerIO writeUInt64:uncompressedDataSize to:data];
-    [StringIO write:thumbnailSHA1 to:data];
+    [StringIO write:[thumbnailBlobKey sha1] to:data];
-    [StringIO write:previewSHA1 to:data];
+    [BooleanIO write:[thumbnailBlobKey stretchEncryptionKey] to:data];
-    [StringIO write:xattrsSHA1 to:data];
+    [StringIO write:[previewBlobKey sha1] to:data];
+    [BooleanIO write:[previewBlobKey stretchEncryptionKey] to:data];
+    [StringIO write:[xattrsBlobKey sha1] to:data];
+    [BooleanIO write:[xattrsBlobKey stretchEncryptionKey] to:data];
     [IntegerIO writeUInt64:xattrsSize to:data];
-    [StringIO write:aclSHA1 to:data];
+    [StringIO write:[aclBlobKey sha1] to:data];
+    [BooleanIO write:[aclBlobKey stretchEncryptionKey] to:data];
     [IntegerIO writeInt32:uid to:data];
     [IntegerIO writeInt32:gid to:data];
     [IntegerIO writeInt32:mode to:data];
@@ -158,4 +182,50 @@
     [IntegerIO writeInt64:st_blocks to:data];
     [IntegerIO writeUInt32:st_blksize to:data];
 }
+- (uint64_t)sizeOnDisk {
+    return (uint64_t)st_blocks * (uint64_t)512;
+}
+
+#pragma mark NSObject
+- (BOOL)isEqual:(id)object {
+    if (![object isKindOfClass:[Node class]]) {
+        return NO;
+    }
+    Node *other = (Node *)object;
+    return treeVersion == [other treeVersion] 
+    && isTree == [other isTree]
+    && uncompressedDataSize == [other uncompressedDataSize]
+    && dataAreCompressed == [other dataAreCompressed]
+    && [dataBlobKeys isEqualToArray:[other dataBlobKeys]]
+    && [NSObject equalObjects:thumbnailBlobKey and:[other thumbnailBlobKey]]
+    && [NSObject equalObjects:previewBlobKey and:[other previewBlobKey]]
+    && xattrsAreCompressed == [other xattrsAreCompressed]
+    && [NSObject equalObjects:xattrsBlobKey and:[other xattrsBlobKey]]
+    && xattrsSize == [other xattrsSize]
+    && aclIsCompressed == [other aclIsCompressed]
+    && [NSObject equalObjects:aclBlobKey and:[other aclBlobKey]]
+    && uid == [other uid]
+    && gid == [other gid]
+    && mode == [other mode]
+    && mtime_sec == [other mtime_sec]
+    && mtime_nsec == [other mtime_nsec]
+    && flags == [other flags]
+    && finderFlags == [other finderFlags]
+    && extendedFinderFlags == [other extendedFinderFlags]
+    && [NSObject equalObjects:finderFileType and:[other finderFileType]]
+    && [NSObject equalObjects:finderFileCreator and:[other finderFileCreator]]
+    && st_dev == [other st_dev]
+    && st_ino == [other st_ino]
+    && st_nlink == [other st_nlink]
+    && st_rdev == [other st_rdev]
+    && ctime_sec == [other ctime_sec]
+    && ctime_nsec == [other ctime_nsec]
+    && createTime_sec == [other createTime_sec]
+    && createTime_nsec == [other createTime_nsec]
+    && st_blocks == [other st_blocks]
+    && st_blksize == [other st_blksize];
+}
+- (NSUInteger)hash {
+    return (NSUInteger)treeVersion + (dataAreCompressed ? 1 : 0) + [dataBlobKeys hash];
+}
 @end

PackIndexEntry.h

@@ -1,34 +1,10 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  PackIndexEntry.h
- 
+//  Arq
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 12/30/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 __MyCompanyName__. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #import <Cocoa/Cocoa.h>
 

PackIndexEntry.m

@@ -1,34 +1,10 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  PackIndexEntry.m
- 
+//  Arq
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 12/30/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 __MyCompanyName__. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #import "PackIndexEntry.h"
 

PackIndexWriter.h

@@ -12,7 +12,11 @@
 @interface PackIndexWriter : NSObject {
     DiskPack *diskPack;
     NSString *destination;
+    uid_t targetUID;
+    gid_t targetGID;
 }
-- (id)initWithPack:(DiskPack *)theDiskPack destination:(NSString *)theDestination;
+- (id)initWithPack:(DiskPack *)theDiskPack destination:(NSString *)theDestination
+         targetUID:(uid_t)theTargetUID 
+         targetGID:(gid_t)theTargetGID;
 - (BOOL)writeIndex:(NSError **)error;
 @end

PackIndexWriter.m

@@ -16,16 +16,22 @@
 #import "SHA1Hash.h"
 #import "NSString_extra.h"
 #import "PackIndexEntry.h"
+#import "BufferedOutputStream.h"
 
 @interface PackIndexWriter (internal)
-- (BOOL)writeEntries:(NSArray *)entries toStream:(id <OutputStream>)os error:(NSError **)error;
+- (BOOL)appendSHA1:(NSString *)theSHA1 error:(NSError **)error;
+- (BOOL)writeEntries:(NSArray *)entries toStream:(BufferedOutputStream *)bos error:(NSError **)error;
 @end
 
 @implementation PackIndexWriter
-- (id)initWithPack:(DiskPack *)theDiskPack destination:(NSString *)theDestination {
+- (id)initWithPack:(DiskPack *)theDiskPack destination:(NSString *)theDestination
+         targetUID:(uid_t)theTargetUID 
+         targetGID:(gid_t)theTargetGID {
     if (self = [super init]) {
         diskPack = [theDiskPack retain];
         destination = [theDestination copy];
+        targetUID = theTargetUID;
+        targetGID = theTargetGID;
     }
     return self;
 }
@@ -39,28 +45,40 @@
     if (entries == nil) {
         return NO;
     }
-    FileOutputStream *fos = [[FileOutputStream alloc] initWithPath:destination append:NO];
+    BufferedOutputStream *bos = [[BufferedOutputStream alloc] initWithPath:destination targetUID:targetUID targetGID:targetGID append:NO];
-    BOOL ret = [self writeEntries:entries toStream:fos error:error];
+    BOOL ret = [self writeEntries:entries toStream:bos error:error];
-    [fos release];
+    if (![bos flush:error]) {
+        ret = NO;
+    }
+    [bos release];
     if (!ret) {
         return NO;
     }
     NSString *indexSHA1 = [SHA1Hash hashFile:destination error:error];
-    NSData *sha1Data = [indexSHA1 hexStringToData];
+    if (![self appendSHA1:indexSHA1 error:error]) {
-    fos = [[FileOutputStream alloc] initWithPath:destination append:YES];
+        return NO;
-    ret = [fos write:[sha1Data bytes] length:[sha1Data length] error:error];
+    }
-    [fos release];
     return ret;
 }
 @end
 
 @implementation PackIndexWriter (internal)
-- (BOOL)writeEntries:(NSArray *)entries toStream:(id <OutputStream>)os error:(NSError **)error {
+- (BOOL)appendSHA1:(NSString *)theSHA1 error:(NSError **)error {
+    NSData *sha1Data = [theSHA1 hexStringToData];
+    BufferedOutputStream *bos = [[BufferedOutputStream alloc] initWithPath:destination targetUID:targetUID targetGID:targetGID append:YES];
+    BOOL ret = [bos writeFully:[sha1Data bytes] length:[sha1Data length] error:error];
+    if (![bos flush:error]) {
+        ret = NO;
+    }
+    [bos release];
+    return ret;
+}
+- (BOOL)writeEntries:(NSArray *)entries toStream:(BufferedOutputStream *)bos error:(NSError **)error {
     // Write header to index.
-    if (![IntegerIO writeUInt32:0xff744f63 to:os error:error]) { // Magic number.
+    if (![IntegerIO writeUInt32:0xff744f63 to:bos error:error]) { // Magic number.
         return NO;
     }
-    if (![IntegerIO writeUInt32:0x00000002 to:os error:error]) { // Version 2.
+    if (![IntegerIO writeUInt32:0x00000002 to:bos error:error]) { // Version 2.
         return NO;
     }
     unsigned int firstByte = 0;
@@ -70,31 +88,31 @@
         NSData *sha1Hex = [[pie objectSHA1] hexStringToData];
         unsigned char myFirstByte = ((unsigned char *)[sha1Hex bytes])[0];
         while ((unsigned int)myFirstByte > firstByte) {
-            if (![IntegerIO writeUInt32:index to:os error:error]) {
+            if (![IntegerIO writeUInt32:index to:bos error:error]) {
                 return NO;
             }
             firstByte++;
         }
     }
     while (firstByte <= 0xff) {
-        if (![IntegerIO writeUInt32:index to:os error:error]) {
+        if (![IntegerIO writeUInt32:index to:bos error:error]) {
             return NO;
         }
         firstByte++;
     }
     for (index = 0; index < [entries count]; index++) {
         PackIndexEntry *pie = [entries objectAtIndex:index];
-        if (![IntegerIO writeUInt64:[pie offset] to:os error:error]
+        if (![IntegerIO writeUInt64:[pie offset] to:bos error:error]
-            || ![IntegerIO writeUInt64:[pie dataLength] to:os error:error]) {
+            || ![IntegerIO writeUInt64:[pie dataLength] to:bos error:error]) {
             return NO;
         }
         // Write sha1 to index.
         NSData *sha1Data = [[pie objectSHA1] hexStringToData];
-        if (![os write:[sha1Data bytes] length:[sha1Data length] error:error]) {
+        if (![bos writeFully:[sha1Data bytes] length:[sha1Data length] error:error]) {
             break;
         }
         // Write 4 bytes (for alignment) to index.
-        if (![IntegerIO writeUInt32:0 to:os error:error]) {
+        if (![IntegerIO writeUInt32:0 to:bos error:error]) {
             return NO;
         }
     }

Restorer.h

@@ -32,18 +32,29 @@
 
 #import <Cocoa/Cocoa.h>
 @class S3Service;
-@class ArqFark;
 @class ArqRepo;
+@class BlobKey;
+@class Commit;
+@class Tree;
 
 @interface Restorer : NSObject {
-    ArqFark *fark;
     ArqRepo *repo;
     NSString *bucketName;
     NSString *rootPath;
+    NSUInteger superUserNodeCount;
     NSMutableArray *restoreNodes;
     NSMutableDictionary *hardlinks;
     unsigned long long writtenToCurrentFile;
+    NSMutableDictionary *errorsByPath;
+    int myUID;
+    int myGID;
+    unsigned long long transferred;
+    unsigned long long total;
+
+    BlobKey *headBlobKey;
+    Commit *head;
+    Tree *headTree;
 }
-- (id)initWithS3Service:(S3Service *)theS3 s3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID bucketName:(NSString *)theBucketName encryptionKey:(NSString *)theEncryptionKey;
+- (id)initWithRepo:(ArqRepo *)theRepo bucketName:(NSString *)theBucketName;
 - (BOOL)restore:(NSError **)error;
 @end

Restorer.m

@@ -33,7 +33,6 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #import "Restorer.h"
-#import "ArqFark.h"
 #import "ArqRepo.h"
 #import "SetNSError.h"
 #import "Tree.h"
@@ -45,214 +44,350 @@
 #import "XAttrSet.h"
 #import "FileOutputStream.h"
 #import "NSFileManager_extra.h"
-#import "CFStreamPair.h"
 #import "NSErrorCodes.h"
 #import "BufferedInputStream.h"
-#import "StreamPairFactory.h"
+#import "BufferedOutputStream.h"
+#import "NSData-Gzip.h"
+#import "GunzipInputStream.h"
+#import "FileACL.h"
 
 #define MAX_RETRIES (10)
 #define MY_BUF_SIZE (8192)
 
 @interface Restorer (internal)
-- (BOOL)addRestoreNodesForTreeSHA1:(NSString *)treeSHA1 relativePath:(NSString *)relativePath error:(NSError **)error;
++ (NSString *)errorDomain;
-- (BOOL)restoreRestoreNode:(RestoreNode *)rn error:(NSError **)error;
+
-- (BOOL)createFile:(Node *)node atPath:(NSString *)path error:(NSError **)error;
+- (BOOL)restoreTree:(Tree *)theTree toPath:(NSString *)thePath error:(NSError **)error;
-- (BOOL)createFileAtPath:(NSString *)path fromSHA1s:(NSArray *)dataSHA1s error:(NSError **)error;
+- (BOOL)restoreNode:(Node *)theNode ofTree:(Tree *)theTree toPath:(NSString *)thePath error:(NSError **)error;
-- (BOOL)appendBlobForSHA1:(NSString *)sha1 toFile:(FileOutputStream *)fos error:(NSError **)error;
+- (BOOL)needSuperUserForTree:(Tree *)theTree;
-- (BOOL)doAppendBlobForSHA1:(NSString *)sha1 toFile:(FileOutputStream *)fos error:(NSError **)error;
+- (BOOL)needSuperUserForTree:(Tree *)theTree node:(Node *)theNode;
+- (BOOL)performSuperUserOps:(NSError **)error;
+- (BOOL)chownNode:(Node *)theNode ofTree:(Tree *)theTree atPath:(NSString *)thePath error:(NSError **)error;
+- (BOOL)chownTree:(Tree *)theTree atPath:(NSString *)thePath error:(NSError **)error;
+- (BOOL)applyUID:(int)theUID gid:(int)theGID mode:(int)theMode rdev:(int)theRdev toPath:(NSString *)thePath error:(NSError **)error;
 - (BOOL)applyTree:(Tree *)tree toPath:(NSString *)restorePath error:(NSError **)error;
 - (BOOL)applyNode:(Node *)node toPath:(NSString *)restorePath error:(NSError **)error;
-- (BOOL)applyACLSHA1:(NSString *)aclSHA1 toFileAttributes:(FileAttributes *)fa error:(NSError **)error;
+- (BOOL)createFile:(Node *)node atPath:(NSString *)path error:(NSError **)error;
-- (BOOL)applyXAttrsSHA1:(NSString *)xattrsSHA1 toFile:(NSString *)path error:(NSError **)error;
+- (BOOL)createFileAtPath:(NSString *)path fromBlobKeys:(NSArray *)dataBlobKeys uncompress:(BOOL)uncompress error:(NSError **)error;
+- (BOOL)appendBlobForBlobKey:(BlobKey *)theBlobKey uncompress:(BOOL)uncompress to:(FileOutputStream *)fos error:(NSError **)error;
+- (BOOL)doAppendBlobForBlobKey:(BlobKey *)theBlobKey uncompress:(BOOL)uncompress to:(BufferedOutputStream *)bos error:(NSError **)error;
 - (BOOL)createSymLink:(Node *)node path:(NSString *)symLinkFile target:(NSString *)target error:(NSError **)error;
+- (BOOL)applyACLBlobKey:(BlobKey *)aclBlobKey uncompress:(BOOL)uncompress toPath:(NSString *)path error:(NSError **)error;
+- (BOOL)applyXAttrsBlobKey:(BlobKey *)xattrsBlobKey uncompress:(BOOL)uncompress toFile:(NSString *)path error:(NSError **)error;
 @end
 
 @implementation Restorer
-- (id)initWithS3Service:(S3Service *)theS3 s3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID bucketName:(NSString *)theBucketName encryptionKey:(NSString *)theEncryptionKey {
+- (id)initWithRepo:(ArqRepo *)theArqRepo bucketName:(NSString *)theBucketName {
     if (self = [super init]) {
-        fark = [[ArqFark alloc] initWithS3Service:theS3 s3BucketName:theS3BucketName computerUUID:theComputerUUID];
+        repo = [theArqRepo retain];
-        repo = [[ArqRepo alloc] initWithS3Service:theS3 s3BucketName:theS3BucketName computerUUID:theComputerUUID bucketUUID:theBucketUUID encryptionKey:theEncryptionKey];
         bucketName = [theBucketName copy];
         rootPath = [[[[NSFileManager defaultManager] currentDirectoryPath] stringByAppendingPathComponent:theBucketName] copy];
         restoreNodes = [[NSMutableArray alloc] init];
         hardlinks = [[NSMutableDictionary alloc] init];
+        errorsByPath = [[NSMutableDictionary alloc] init];
+        myUID = geteuid();
+        myGID = getgid();
     }
     return self;
 }
 - (void)dealloc {
-    [fark release];
     [repo release];
     [bucketName release];
     [rootPath release];
     [restoreNodes release];
     [hardlinks release];
+    [errorsByPath release];
+    [headBlobKey release];
+    [head release];
+    [headTree release];
     [super dealloc];
 }
 - (BOOL)restore:(NSError **)error {
     if ([[NSFileManager defaultManager] fileExistsAtPath:rootPath]) {
-        SETNSERROR(@"RestorerErrorDomain", -1, @"%@ already exists", rootPath);
+        SETNSERROR([Restorer errorDomain], -1, @"%@ already exists", rootPath);
         return NO;
     }
     if (![[NSFileManager defaultManager] createDirectoryAtPath:rootPath withIntermediateDirectories:YES attributes:nil error:error]) {
         HSLogError(@"failed to create directory %@", rootPath);
         return NO;
     }
-    NSString *headSHA1 = [repo headSHA1:error];
+    headBlobKey = [[repo headBlobKey:error] retain];
-    if (headSHA1 == nil) {
+    if (headBlobKey == nil) {
+        SETNSERROR([Restorer errorDomain], -1, @"no backup found");
         return NO;
     }
-    if (headSHA1 == nil) {
+    head = [[repo commitForBlobKey:headBlobKey error:error] retain];
-        SETNSERROR(@"RestorerErrorDomain", -1, @"no backup found");
-        return NO;
-    }
-    Commit *head = [repo commitForSHA1:headSHA1 error:error];
     if (head == nil) {
         return NO;
     }
-    if (![self addRestoreNodesForTreeSHA1:[head treeSHA1] relativePath:@"" error:error]) {
+    headTree = [[repo treeForBlobKey:[head treeBlobKey] error:error] retain];
+    if (headTree == nil) {
         return NO;
     }
-    for (RestoreNode *rn in restoreNodes) {
+    if (![self restoreTree:headTree toPath:rootPath error:error]) {
-        NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
+        return NO;
-        NSError *myError = nil;
-        BOOL ret = [self restoreRestoreNode:rn error:&myError];
-        [myError retain];
-        [pool drain];
-        [myError autorelease];
-        if (!ret) {
-            if (error != NULL) {
-                *error = myError;
-            }
-            return NO;
-        }
     }
     return YES;
 }
 @end
 
 @implementation Restorer (internal)
-- (BOOL)addRestoreNodesForTreeSHA1:(NSString *)treeSHA1 relativePath:(NSString *)relativePath error:(NSError **)error {
++ (NSString *)errorDomain {
-    Tree *tree = [repo treeForSHA1:treeSHA1 error:error];
+    return @"RestorerErrorDomain";
-    if (tree == nil) {
+}
-        return NO;
+
+- (BOOL)restoreTree:(Tree *)theTree toPath:(NSString *)thePath error:(NSError **)error {
+    NSNumber *inode = [NSNumber numberWithInt:[theTree st_ino]];
+    NSString *existing = nil;
+    if ([theTree st_nlink] > 1) {
+        existing = [hardlinks objectForKey:inode];
     }
-    RestoreNode *treeRN = [[RestoreNode alloc] initWithTree:tree nodeName:nil relativePath:relativePath];
+    if (existing != nil) {
-    [restoreNodes addObject:treeRN];
+        // Link.
-    [treeRN release];
+        if (link([existing fileSystemRepresentation], [thePath fileSystemRepresentation]) == -1) {
-    for (NSString *childNodeName in [tree childNodeNames]) {
+            int errnum = errno;
-        Node *childNode = [tree childNodeWithName:childNodeName];
+            SETNSERROR([Restorer errorDomain], -1, @"link(%@,%@): %s", existing, thePath, strerror(errnum));
-        NSString *childRelativePath = [NSString stringWithFormat:@"%@/%@", relativePath, childNodeName];
+            HSLogError(@"link() failed");
-        if ([childNode isTree]) {
+            return NO;
-            if (![self addRestoreNodesForTreeSHA1:[childNode treeSHA1] relativePath:childRelativePath error:error]) {
+        }
-                return NO;
+    } else {
+        if (![[NSFileManager defaultManager] fileExistsAtPath:thePath] 
+            && ![[NSFileManager defaultManager] createDirectoryAtPath:thePath withIntermediateDirectories:YES attributes:nil error:error]) {
+            return NO;
+        }
+        NSAutoreleasePool *pool = nil;
+        BOOL ret = YES;
+        for (NSString *childNodeName in [theTree childNodeNames]) {
+            [pool drain];
+            pool = [[NSAutoreleasePool alloc] init];
+            Node *childNode = [theTree childNodeWithName:childNodeName];
+            NSString *childPath = [thePath stringByAppendingPathComponent:childNodeName];
+            if ([childNode isTree]) {
+                Tree *childTree = [repo treeForBlobKey:[childNode treeBlobKey] error:error];
+                if (childTree == nil) {
+                    ret = NO;
+                    break;
+                }
+                NSError *restoreError = nil;
+                if (![self restoreTree:childTree toPath:childPath error:&restoreError]) {
+                    HSLogDebug(@"error restoring %@: %@", childPath, restoreError);
+                    if ([restoreError isErrorWithDomain:[Restorer errorDomain] code:ERROR_ABORT_REQUESTED]) {
+                        ret = NO;
+                        if (error != NULL) {
+                            *error = restoreError;
+                        }
+                        break;
+                    }
+                    [self performSelectorOnMainThread:@selector(addError:) withObject:[NSArray arrayWithObjects:restoreError, childPath, nil] waitUntilDone:YES];
+                }
+            } else {
+                NSError *restoreError = nil;
+                if (![self restoreNode:childNode ofTree:theTree toPath:childPath error:&restoreError]) {
+                    if ([restoreError isErrorWithDomain:[Restorer errorDomain] code:ERROR_ABORT_REQUESTED]) {
+                        ret = NO;
+                        if (error != NULL) {
+                            *error = restoreError;
+                        }
+                        break;
+                    }
+                    HSLogDebug(@"error restoring %@: %@", childPath, restoreError);
+                    [self performSelectorOnMainThread:@selector(addError:) withObject:[NSArray arrayWithObjects:restoreError, childPath, nil] waitUntilDone:YES];
+                }
             }
-        } else {
+        }
-            RestoreNode *childRN = [[RestoreNode alloc] initWithTree:tree nodeName:childNodeName relativePath:childRelativePath];
+        if (error != NULL) { [*error retain]; }
-            [restoreNodes addObject:childRN];
+        [pool drain];
-            [childRN release];
+        if (error != NULL) { [*error autorelease]; }
+        if (!ret) {
+            return NO;
+        }
+        
+        if (![self applyTree:theTree toPath:thePath error:error]) {
+            return NO;
+        }
+        [hardlinks setObject:thePath forKey:inode];
+        if ([self needSuperUserForTree:theTree]) {
+            superUserNodeCount++;
         }
     }
     return YES;
 }
-- (BOOL)restoreRestoreNode:(RestoreNode *)rn error:(NSError **)error {
+- (BOOL)restoreNode:(Node *)theNode ofTree:(Tree *)theTree toPath:(NSString *)thePath error:(NSError **)error {
-    printf("restoring %s%s\n", [bucketName UTF8String], [[rn relativePath] UTF8String]);
+    NSAssert(theNode != nil, @"theNode can't be nil");
-    NSString *restorePath = [rootPath stringByAppendingPathComponent:[rn relativePath]];
+    NSAssert(theTree != nil, @"theTree can't be nil");
-    NSString *parentPath = [restorePath stringByDeletingLastPathComponent];
+    
-    if (![[NSFileManager defaultManager] fileExistsAtPath:parentPath] 
+    NSNumber *inode = [NSNumber numberWithInt:[theNode st_ino]];
-        && ![[NSFileManager defaultManager] createDirectoryAtPath:parentPath withIntermediateDirectories:YES attributes:nil error:error]) {
+    NSString *existing = nil;
-        HSLogError(@"failed to create directory %@", parentPath);
+    if ([theNode st_nlink] > 1) {
-        return NO;
+        existing = [hardlinks objectForKey:inode];
     }
-    BOOL createdFile = NO;
+    if (existing != nil) {
-    int nlink = [rn node] == nil ? [[rn tree] st_nlink] : [[rn node] st_nlink];
+        // Link.
-    if (nlink > 1) {
+        if (link([existing fileSystemRepresentation], [thePath fileSystemRepresentation]) == -1) {
-        int ino = [rn node] == nil ? [[rn tree] st_ino] : [[rn node] st_ino];
+            int errnum = errno;
-        NSNumber *inode = [NSNumber numberWithInt:ino];
+            SETNSERROR([Restorer errorDomain], -1, @"link(%@,%@): %s", existing, thePath, strerror(errnum));
-        RestoreNode *existing = [hardlinks objectForKey:inode];
+            HSLogError(@"link() failed");
-        if (existing != nil) {
-            // Link.
-            NSString *existingPath = [rootPath stringByAppendingPathComponent:[existing relativePath]];
-            if (([existing node] == nil) != ([rn node] == nil)) {
-                SETNSERROR(@"RestoreErrorDomain", -1, @"cannot link a directory to a file");
-                HSLogError(@"can't link directory to a file");
-                return NO;
-            }
-            if (link([existingPath fileSystemRepresentation], [restorePath fileSystemRepresentation]) == -1) {
-                SETNSERROR(@"RestoreErrorDomain", -1, @"link(%@,%@): %s", existingPath, restorePath, strerror(errno));
-                HSLogError(@"link() failed");
-                return NO;
-            }
-            createdFile = YES;
-        } else {
-            [hardlinks setObject:rn forKey:inode];
-        }
-    }
-    if (!createdFile) {
-        Node *node = [rn node];
-        if (node == nil) {
-            Tree *tree = [rn tree];
-            if (![[NSFileManager defaultManager] fileExistsAtPath:restorePath] && ![[NSFileManager defaultManager] createDirectoryAtPath:restorePath withIntermediateDirectories:NO attributes:nil error:error]) {
-                HSLogError(@"error creating %@", restorePath);
-                return NO;
-            }
-            if (![self applyTree:tree toPath:restorePath error:error]) {
-                HSLogError(@"applyTree error");
-                return NO;
-            }
-        } else {
-            int mode = [node mode];
-            BOOL isFifo = (mode & S_IFIFO) == S_IFIFO;
-            if (isFifo) {
-                if (mkfifo([restorePath fileSystemRepresentation], mode) == -1) {
-                    SETNSERROR(@"RestoreErrorDomain", errno, @"mkfifo(%@): %s", restorePath, strerror(errno));
-                    return NO;
-                }
-                if (![self applyNode:node toPath:restorePath error:error]) {
-                    HSLogError(@"applyNode error");
-                    return NO;
-                }
-            } else if ((mode & S_IFSOCK) == S_IFSOCK) {
-                // Skip socket -- restoring it doesn't make any sense.
-            } else if ((mode & S_IFREG) == 0 && ((mode & S_IFCHR) == S_IFCHR || (mode & S_IFBLK) == S_IFBLK)) {
-                if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:restorePath error:error]) {
-                    return NO;
-                }
-                if (mknod([restorePath fileSystemRepresentation], mode, [node st_rdev]) == -1) {
-                    SETNSERROR(@"RestorerErrorDomain", -1, @"mknod(%@): %s", restorePath, strerror(errno));
-                    return NO;
-                }
-            } else {
-                if (![self createFile:node atPath:restorePath error:error]) {
-                    HSLogError(@"createFile error");
-                    return NO;
-                }
-                if (![self applyNode:node toPath:restorePath error:error]) {
-                    HSLogError(@"applyNode error");
-                    return NO;
-                }
-            }
-        }
-        FileAttributes *fa = [[[FileAttributes alloc] initWithPath:restorePath error:error] autorelease];
-        if (fa == nil) {
             return NO;
         }
-        int flags = [fa flags];
+    } else {
-        if (flags) {
+        int mode = [theNode mode];
-            // Clear the flags temporarily so we can change ownership of the file.
+        if (S_ISFIFO(mode)) {
-            if (![fa applyFlags:0 error:error]) {
+            if (mkfifo([thePath fileSystemRepresentation], mode) == -1) {
+                int errnum = errno;
+                SETNSERROR([Restorer errorDomain], errnum, @"mkfifo(%@): %s", thePath, strerror(errnum));
+                return NO;
+            }
+            if (![self applyNode:theNode toPath:thePath error:error]) {
+                HSLogError(@"applyNode error");
+                return NO;
+            }
+        } else if (S_ISSOCK(mode)) {
+            // Skip socket -- restoring it doesn't make any sense.
+        } else if (S_ISCHR(mode)) {
+            // character device: needs to be done as super-user.
+        } else if (S_ISBLK(mode)) {
+            // block device: needs to be done as super-user.
+        } else {
+            if (![self createFile:theNode atPath:thePath error:error]) {
+                HSLogError(@"createFile error");
+                return NO;
+            }
+            if (![self applyNode:theNode toPath:thePath error:error]) {
+                HSLogError(@"applyNode error");
                 return NO;
             }
         }
-        int uid = [rn node] == nil ? [[rn tree] uid] : [[rn node] uid];
+        [hardlinks setObject:thePath forKey:inode];
-        int gid = [rn node] == nil ? [[rn tree] gid] : [[rn node] gid];
+        if ([self needSuperUserForTree:theTree node:theNode]) {
-        NSError *chownError;
+            superUserNodeCount++;
-        if (![fa applyUID:uid gid:gid error:&chownError]) {
-            fprintf(stderr, "error applying UID and GID to %s: %s\n", [restorePath fileSystemRepresentation], [[chownError localizedDescription] UTF8String]);
         }
-        if (flags) {
+    }    
-            if (![fa applyFlags:flags error:error]) {
+    return YES;
-                return NO;
+}
+- (BOOL)needSuperUserForTree:(Tree *)theTree {
+    NSAssert(theTree != nil, @"theTree can't be nil");
+    
+    int uid = [theTree uid];
+    int gid = [theTree gid];
+    int mode = [theTree mode];
+    if ((uid != myUID) || (gid != myGID)) {
+        return YES;
+    }
+    if (mode & (S_ISUID|S_ISGID|S_ISVTX)) {
+        return YES;
+    }
+    return NO;
+}
+- (BOOL)needSuperUserForTree:(Tree *)theTree node:(Node *)theNode {
+    NSAssert(theNode != nil, @"theNode can't be nil");
+    NSAssert(theTree != nil, @"theTree can't be nil");
+    
+    int uid = [theNode uid];
+    int gid = [theNode gid];
+    int mode = [theNode mode];
+    if ([theTree treeVersion] >= 7 && (S_ISCHR(mode) || S_ISBLK(mode))) {
+        return YES;
+    }
+    if ((uid != myUID) || (gid != myGID)) {
+        return YES;
+    }
+    if (mode & (S_ISUID|S_ISGID|S_ISVTX)) {
+        return YES;
+    }
+    return NO;
+}
+- (BOOL)performSuperUserOps:(NSError **)error {
+    return [self chownTree:headTree atPath:rootPath error:error];
+}
+- (BOOL)chownNode:(Node *)theNode ofTree:(Tree *)theTree atPath:(NSString *)thePath error:(NSError **)error {
+    if ([[errorsByPath allKeys] containsObject:thePath]) {
+        HSLogDebug(@"error restoring %@; skipping chownNode", thePath);
+        return YES;
+    }
+    if ([self needSuperUserForTree:theTree node:theNode]) {
+        if (![self applyUID:[theNode uid] gid:[theNode gid] mode:[theNode mode] rdev:[theNode st_rdev] toPath:thePath error:error]) {
+            return NO;
+        }
+    }
+    return YES;
+}
+- (BOOL)chownTree:(Tree *)theTree atPath:(NSString *)thePath error:(NSError **)error {
+    if ([[errorsByPath allKeys] containsObject:thePath]) {
+        HSLogDebug(@"error restoring %@; skipping chownTree", thePath);
+        return YES;
+    }
+    
+    NSAutoreleasePool *pool = nil;
+    BOOL ret = YES;
+    for (NSString *childNodeName in [theTree childNodeNames]) {
+        [pool drain];
+        pool = [[NSAutoreleasePool alloc] init];
+        Node *childNode = [theTree childNodeWithName:childNodeName];
+        NSString *childPath = [thePath stringByAppendingPathComponent:childNodeName];
+        if ([childNode isTree]) {
+            Tree *childTree = [repo treeForBlobKey:[childNode treeBlobKey] error:error];
+            if (childTree == nil) {
+                ret = NO;
+                break;
             }
+            if (![self chownTree:childTree atPath:childPath error:error]) {
+                ret = NO;
+                break;
+            }
+        } else {
+            if (![self chownNode:childNode ofTree:theTree atPath:childPath error:error]) {
+                ret = NO;
+                break;
+            }
+        }
+    }
+    if (error != NULL) { [*error retain]; }
+    [pool drain];
+    if (error != NULL) { [*error autorelease]; }
+    if (!ret) {
+        return NO;
+    }
+    
+    if ([self needSuperUserForTree:theTree]) {
+        if (![self applyUID:[theTree uid] gid:[theTree gid] mode:[theTree mode] rdev:[theTree st_rdev] toPath:thePath error:error]) {
+            return NO;
+        }
+    }
+    return YES;
+}
+- (BOOL)applyUID:(int)theUID gid:(int)theGID mode:(int)theMode rdev:(int)theRdev toPath:(NSString *)thePath error:(NSError **)error {
+    if (S_ISCHR(theMode) || S_ISBLK(theMode)) {
+        if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:thePath error:error]) {
+            return NO;
+        }
+        HSLogDebug(@"mknod(%@, %d, %d)", thePath, theMode, theRdev);
+        if (mknod([thePath fileSystemRepresentation], theMode, theRdev) == -1) {
+            int errnum = errno;
+            HSLogError(@"mknod(%@) error %d: %s", thePath, errnum, strerror(errnum));
+            SETNSERROR([Restorer errorDomain], -1, @"failed to make device node %@: %s", thePath, strerror(errnum));
+            return NO;
+        }
+    }
+    FileAttributes *fa = [[[FileAttributes alloc] initWithPath:thePath error:error] autorelease];
+    if (fa == nil) {
+        return NO;
+    }
+    int flags = [fa flags];
+    if (flags) {
+        // Clear the flags temporarily so we can change ownership of the file.
+        if (![fa applyFlags:0 error:error]) {
+            return NO;
+        }
+    }
+    if (![fa applyMode:theMode error:error]) {
+        return NO;
+    }
+    if (![fa applyUID:theUID gid:theGID error:error]) {
+        return NO;
+    }
+    if (flags) {
+        if (![fa applyFlags:flags error:error]) {
+            return NO;
         }
     }
     return YES;
@@ -262,20 +397,17 @@
     if (!fa) {
         return NO;
     }
+    if (![self applyXAttrsBlobKey:[tree xattrsBlobKey] uncompress:[tree xattrsAreCompressed] toFile:path error:error]) {
+        return NO;
+    }
     if (![fa applyFinderFlags:[tree finderFlags] error:error]
         || ![fa applyExtendedFinderFlags:[tree extendedFinderFlags] error:error]) {
         return NO;
     }
-    if (![self applyACLSHA1:[tree aclSHA1] toFileAttributes:fa error:error]) {
+    if (([tree mode] & (S_ISUID|S_ISGID|S_ISVTX)) && ![fa applyMode:[tree mode] error:error]) {
         return NO;
     }
-    if (![self applyXAttrsSHA1:[tree xattrsSHA1] toFile:path error:error]) {
+    if (!S_ISLNK([tree mode]) && [tree treeVersion] >= 7 && ![fa applyMTimeSec:tree.mtime_sec mTimeNSec:tree.mtime_nsec error:error]) {
-        return NO;
-    }
-    if (([tree mode] & (S_ISUID|S_ISGID|S_ISVTX) != 0) && ![fa applyMode:[tree mode] error:error]) {
-        return NO;
-    }
-    if (([tree mode] & S_IFLNK) != S_IFLNK && [tree treeVersion] >= 7 && ![fa applyMTimeSec:tree.mtime_sec mTimeNSec:tree.mtime_nsec error:error]) {
         return NO;
     }
     if (([tree treeVersion] >= 7) && ![fa applyCreateTimeSec:tree.createTime_sec createTimeNSec:tree.createTime_nsec error:error]) {
@@ -284,6 +416,9 @@
     if (![fa applyFlags:[tree flags] error:error]) {
         return NO;
     }
+    if (![self applyACLBlobKey:[tree aclBlobKey] uncompress:[tree aclIsCompressed] toPath:path error:error]) {
+        return NO;
+    }
     return YES;
 }
 - (BOOL)applyNode:(Node *)node toPath:(NSString *)path error:(NSError **)error {
@@ -291,28 +426,31 @@
     if (!fa) {
         return NO;
     }
-    if (![self applyACLSHA1:[node aclSHA1] toFileAttributes:fa error:error]) {
+    if (![self applyXAttrsBlobKey:[node xattrsBlobKey] uncompress:[node xattrsAreCompressed] toFile:path error:error]) {
         return NO;
     }
-    BOOL isFifo = ([node mode] & S_IFIFO) == S_IFIFO;
+    if (![self applyACLBlobKey:[node aclBlobKey] uncompress:[node aclIsCompressed] toPath:path error:error]) {
-    if (!isFifo) {
+        return NO;
+    }
+    if (!S_ISFIFO([node mode])) {
         if (![fa applyFinderFlags:[node finderFlags] error:error]
             || ![fa applyExtendedFinderFlags:[node extendedFinderFlags] error:error]
-            || ![self applyXAttrsSHA1:[node xattrsSHA1] toFile:path error:error]
             || ![fa applyFinderFileType:[node finderFileType] finderFileCreator:[node finderFileCreator] error:error]) {
             return NO;
         }
     }
-    if (([node mode] & (S_ISUID|S_ISGID|S_ISVTX) != 0) && ![fa applyMode:[node mode] error:error]) {
+    if (!([node mode] & (S_ISUID|S_ISGID|S_ISVTX))) {
-        return NO;
+        if (![fa applyMode:[node mode] error:error]) {
+            return NO;
+        }
     }
-    if (([node mode] & S_IFLNK) != S_IFLNK && [node treeVersion] >= 7 && ![fa applyMTimeSec:node.mtime_sec mTimeNSec:node.mtime_nsec error:error]) {
+    if (!S_ISLNK([node mode]) && [node treeVersion] >= 7 && ![fa applyMTimeSec:node.mtime_sec mTimeNSec:node.mtime_nsec error:error]) {
         return NO;
     }
     if (([node treeVersion] >= 7) && ![fa applyCreateTimeSec:node.createTime_sec createTimeNSec:node.createTime_nsec error:error]) {
         return NO;
     }
-    if (!isFifo) {
+    if (!S_ISFIFO([node mode])) {
         if (![fa applyFlags:[node flags] error:error]) {
             return NO;
         }
@@ -330,39 +468,53 @@
             return NO;
         }
     }
-    HSLogTrace(@"%qu bytes -> %@", [node dataSize], path);
+    HSLogTrace(@"%qu bytes -> %@", [node uncompressedDataSize], path);
-    if (([node mode] & S_IFLNK) == S_IFLNK) {
+    if (S_ISLNK([node mode])) {
-        NSData *data = [repo blobDataForSHA1s:[node dataSHA1s] error:error];
+        NSMutableData *data = [NSMutableData data];
-        if (data == nil) {
+        for (BlobKey *dataBlobKey in [node dataBlobKeys]) {
-            HSLogError(@"error getting data for %@", [node dataSHA1s]);
+            NSData *blobData = [repo blobDataForBlobKey:dataBlobKey error:error];
-            return NO;
+            if (blobData == nil) {
+                HSLogError(@"error getting data for %@", dataBlobKey);
+                return NO;
+            }
+            if ([node dataAreCompressed]) {
+                blobData = [blobData gzipInflate];
+            }
+            [data appendData:blobData];
         }
         NSString *target = [[[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] autorelease];
         if (![self createSymLink:node path:path target:target error:error]) {
             HSLogError(@"error creating sym link %@", path);
             return NO;
         }
-    } else if ([node dataSize] > 0) {
+    } else if ([node uncompressedDataSize] > 0) {
-        if (![self createFileAtPath:path fromSHA1s:[node dataSHA1s] error:error]) {
+        if (![self createFileAtPath:path fromBlobKeys:[node dataBlobKeys] uncompress:[node dataAreCompressed] error:error]) {
+            NSError *myError = nil;
+            if ([[NSFileManager defaultManager] fileExistsAtPath:path] && ![[NSFileManager defaultManager] removeItemAtPath:path error:&myError]) {
+                HSLogError(@"error deleting incorrectly-restored file %@: %@", path, myError);
+            }
             return NO;
         }
     } else {
         // It's a zero-byte file.
-        int fd = open([path fileSystemRepresentation], O_CREAT|O_EXCL, [node mode]);
+        int fd = open([path fileSystemRepresentation], O_CREAT|O_EXCL, S_IRWXU);
         if (fd == -1) {
-            SETNSERROR(@"UnixErrorDomain", errno, @"%s: %@", strerror(errno), path);
+            int errnum = errno;
-            HSLogError(@"error opening %@", path);
+            HSLogError(@"open(%@) error %d: %s", path, errnum, strerror(errnum));
+            SETNSERROR(@"UnixErrorDomain", errnum, @"failed to open %@: %s", path, strerror(errnum));
             return NO;
         }
         close(fd);
     }
+    HSLogDetail(@"restored %@", path);
     return YES;
 }
-- (BOOL)createFileAtPath:(NSString *)path fromSHA1s:(NSArray *)dataSHA1s error:(NSError **)error {
+- (BOOL)createFileAtPath:(NSString *)path fromBlobKeys:(NSArray *)dataBlobKeys uncompress:(BOOL)uncompress error:(NSError **)error {
     FileOutputStream *fos = [[FileOutputStream alloc] initWithPath:path append:NO];
     BOOL ret = YES;
-    for (NSString *sha1 in dataSHA1s) {
+    writtenToCurrentFile = 0;
-        if (![self appendBlobForSHA1:sha1 toFile:fos error:error]) {
+    for (BlobKey *dataBlobKey in dataBlobKeys) {
+        if (![self appendBlobForBlobKey:dataBlobKey uncompress:uncompress to:fos error:error]) {
             ret = NO;
             break;
         }
@@ -370,26 +522,33 @@
     [fos release];
     return ret;
 }
-- (BOOL)appendBlobForSHA1:(NSString *)sha1 toFile:(FileOutputStream *)fos error:(NSError **)error {
+- (BOOL)appendBlobForBlobKey:(BlobKey *)theBlobKey uncompress:(BOOL)uncompress to:(FileOutputStream *)fos error:(NSError **)error {
-    int i = 0;
     BOOL ret = NO;
     NSError *myError = nil;
     NSAutoreleasePool *pool = nil;
+    unsigned long long transferredSoFar = transferred;
+    unsigned long long writtenToCurrentFileSoFar = writtenToCurrentFile;
     for (;;) {
         [pool drain];
         pool = [[NSAutoreleasePool alloc] init];
-        if ([self doAppendBlobForSHA1:sha1 toFile:fos error:&myError]) {
+        BufferedOutputStream *bos = [[[BufferedOutputStream alloc] initWithUnderlyingOutputStream:fos] autorelease];
+        if ([self doAppendBlobForBlobKey:theBlobKey uncompress:uncompress to:bos error:&myError] && [bos flush:&myError]) {
             ret = YES;
             break;
         }
-        [[StreamPairFactory theFactory] clear];
+        if ([myError isErrorWithDomain:[Restorer errorDomain] code:ERROR_ABORT_REQUESTED]) {
-        BOOL isNetworkError = [[myError domain] isEqualToString:[CFStreamPair errorDomain]];
+            HSLogInfo(@"restore canceled");
-        // Retry indefinitely on network errors:
-        if (!isNetworkError && (++i >= MAX_RETRIES)) {
-            HSLogError(@"failed to get blob %@ after %d retries: %@", sha1, i, [myError localizedDescription]);
             break;
         }
-        HSLogWarn(@"error appending blob %@ to file %@ (retrying): %@", sha1, [fos path], [myError localizedDescription]);
+        if (![myError isTransientError]) {
+            HSLogError(@"error getting appending blob %@ to %@: %@", theBlobKey, bos, myError);
+            ret = NO;
+            break;
+        }
+        HSLogWarn(@"error appending blob %@ to %@ (retrying): %@", theBlobKey, bos, [myError localizedDescription]);
+        // Reset transferred:
+        transferred = transferredSoFar;
+        writtenToCurrentFile = writtenToCurrentFileSoFar;
         // Seek back to the starting offset for this blob:
         if (![fos seekTo:writtenToCurrentFile error:&myError]) {
             ret = NO;
@@ -404,22 +563,24 @@
     }
     return ret;
 }
-- (BOOL)doAppendBlobForSHA1:(NSString *)sha1 toFile:(FileOutputStream *)fos error:(NSError **)error {
+- (BOOL)doAppendBlobForBlobKey:(BlobKey *)theBlobKey uncompress:(BOOL)uncompress to:(BufferedOutputStream *)bos error:(NSError **)error {
-    if (error != NULL) {
+    ServerBlob *sb = [[repo newServerBlobForBlobKey:theBlobKey error:error] autorelease];
-        *error = nil;
-    }
-    ServerBlob *sb = [[repo newServerBlobForSHA1:sha1 error:error] autorelease];
     if (sb == nil) {
         return NO;
     }
     id <InputStream> is = [[sb newInputStream] autorelease];
+    if (uncompress) {
+        is = [[[GunzipInputStream alloc] initWithUnderlyingStream:is] autorelease];
+    }
+    HSLogDebug(@"writing %@ to %@", is, bos);
     BOOL ret = YES;
+    NSError *myError = nil;
     NSAutoreleasePool *pool = nil;
     unsigned char *buf = (unsigned char *)malloc(MY_BUF_SIZE);
     for (;;) {
         [pool drain];
         pool = [[NSAutoreleasePool alloc] init];
-        NSUInteger received = [is read:buf bufferLength:MY_BUF_SIZE error:error];
+        NSInteger received = [is read:buf bufferLength:MY_BUF_SIZE error:&myError];
         if (received < 0) {
             ret = NO;
             break;
@@ -427,19 +588,20 @@
         if (received == 0) {
             break;
         }
-        if (![fos write:buf length:received error:error]) {
+        if (![bos writeFully:buf length:received error:error]) {
             ret = NO;
             break;
         }
+        
+        transferred += received;
         writtenToCurrentFile += received;
     }
     free(buf);
-    if (error != NULL) {
+    [myError retain];
-        [*error retain];
-    }
     [pool drain];
+    [myError autorelease];
     if (error != NULL) {
-        [*error autorelease];
+        *error = myError;
     }
     return ret;
 }
@@ -451,35 +613,52 @@
         }
     }
     if (symlink([target fileSystemRepresentation], [symLinkFile fileSystemRepresentation]) == -1) {
-        SETNSERROR(@"UnixErrorDomain", errno, @"symlink(%@): %s", symLinkFile, strerror(errno));
+        int errnum = errno;
+        HSLogError(@"symlink(%@, %@) error %d: %s", target, symLinkFile, errnum, strerror(errnum));
+        SETNSERROR(@"UnixErrorDomain", errnum, @"failed to create symlink %@ to %@: %s", symLinkFile, target, strerror(errnum));
         return NO;
     }
     return YES;
 }
-- (BOOL)applyACLSHA1:(NSString *)aclSHA1 toFileAttributes:(FileAttributes *)fa error:(NSError **)error {
+- (BOOL)applyACLBlobKey:(BlobKey *)aclBlobKey uncompress:(BOOL)uncompress toPath:(NSString *)path error:(NSError **)error {
-    if (aclSHA1 != nil) {
+    if (aclBlobKey != nil) {
-        NSData *data = [repo blobDataForSHA1:aclSHA1 error:error];
+        NSData *data = [repo blobDataForBlobKey:aclBlobKey error:error];
         if (data == nil) {
             return NO;
         }
+        if (uncompress) {
+            data = [data gzipInflate];
+        }
         NSString *aclString = [[[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] autorelease];
-        if (![fa applyAcl:aclString error:error]) {
+        
+        NSString *currentAclString = nil;
+        if (![FileACL aclText:&currentAclString forFile:path error:error]) {
             return NO;
         }
+        if (![currentAclString isEqualToString:aclString] && [aclString length] > 0) {
+            if (![FileACL writeACLText:aclString toFile:path error:error]) {
+                return NO;
+            }
+        }
     }
     return YES;
 }
-- (BOOL)applyXAttrsSHA1:(NSString *)xattrsSHA1 toFile:(NSString *)path error:(NSError **)error {
+- (BOOL)applyXAttrsBlobKey:(BlobKey *)xattrsBlobKey uncompress:(BOOL)uncompress toFile:(NSString *)path error:(NSError **)error {
-    if (xattrsSHA1 != nil) {
+    if (xattrsBlobKey != nil) {
-        NSData *xattrsData = [repo blobDataForSHA1:xattrsSHA1 error:error];
+        NSData *xattrsData = [repo blobDataForBlobKey:xattrsBlobKey error:error];
         if (xattrsData == nil) {
             return NO;
         }
-        DataInputStream *is = [xattrsData newInputStream];
+        id <InputStream> is = [xattrsData newInputStream];
+        if (uncompress) {
+            id <InputStream> uncompressed = [[GunzipInputStream alloc] initWithUnderlyingStream:is];
+            [is release];
+            is = uncompressed;
+        }
         BufferedInputStream *bis = [[BufferedInputStream alloc] initWithUnderlyingStream:is];
+        [is release];
         XAttrSet *set = [[[XAttrSet alloc] initWithBufferedInputStream:bis error:error] autorelease];
         [bis release];
-        [is release];
         if (!set) {
             return NO;
         }

Tree.h

@@ -1,48 +1,27 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  Tree.h
- 
+//  Backup
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 3/25/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 PhotoMinds LLC. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #import <Cocoa/Cocoa.h>
 #import "Blob.h"
 @class BufferedInputStream;
 @class Node;
+@class BlobKey;
 
-#define CURRENT_TREE_VERSION 10
+#define CURRENT_TREE_VERSION 14
 #define TREE_HEADER_LENGTH (8)
 
 @interface Tree : NSObject {
     int treeVersion;
-	NSString *xattrsSHA1;
+    BOOL xattrsAreCompressed;
+    BlobKey *xattrsBlobKey;
     unsigned long long xattrsSize;
-	NSString *aclSHA1;
+    BOOL aclIsCompressed;
+    BlobKey *aclBlobKey;
     int uid;
     int gid;
 	int mode;
@@ -61,6 +40,7 @@
     int64_t createTime_nsec;
     int64_t st_blocks;
     uint32_t st_blksize;
+    uint64_t aggregateSizeOnDisk;
 	NSMutableDictionary *nodes;
 }
 + (NSString *)errorDomain;
@@ -68,11 +48,14 @@
 - (NSArray *)childNodeNames;
 - (Node *)childNodeWithName:(NSString *)name;
 - (BOOL)containsNodeNamed:(NSString *)name;
+- (NSDictionary *)nodes;
 - (Blob *)toBlob;
 
-@property(readonly,copy) NSString *xattrsSHA1;
+@property(readonly) BOOL xattrsAreCompressed;
+@property(readonly,copy) BlobKey *xattrsBlobKey;
 @property(readonly) unsigned long long xattrsSize;
-@property(readonly,copy) NSString *aclSHA1;
+@property(readonly) BOOL aclIsCompressed;
+@property(readonly,copy) BlobKey *aclBlobKey;
 @property(readonly) int uid;
 @property(readonly) int gid;
 @property(readonly) int mode;
@@ -81,6 +64,7 @@
 @property(readonly) long long flags;
 @property(readonly) int finderFlags;
 @property(readonly) int extendedFinderFlags;
+@property(readonly) int st_dev;
 @property(readonly) int treeVersion;
 @property(readonly) int st_rdev;
 @property(readonly) long long ctime_sec;
@@ -89,5 +73,7 @@
 @property(readonly) long long createTime_nsec;
 @property(readonly) uint32_t st_nlink;
 @property(readonly) int st_ino;
-
+@property(readonly) int64_t st_blocks;
+@property(readonly) uint32_t st_blksize;
+@property(readonly) uint64_t aggregateSizeOnDisk;
 @end

Tree.m

@@ -1,34 +1,10 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  Tree.m
- 
+//  Backup
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 3/25/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 PhotoMinds LLC. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #import "StringIO.h"
 #import "IntegerIO.h"
@@ -37,31 +13,56 @@
 #import "Tree.h"
 #import "Blob.h"
 #import "DataInputStream.h"
-#import "BufferedInputStream.h"
 #import "SetNSError.h"
 #import "RegexKitLite.h"
 #import "NSErrorCodes.h"
+#import "BufferedInputStream.h"
+#import "NSData-Gzip.h"
+#import "GunzipInputStream.h"
+#import "BlobKey.h"
+#import "NSObject_extra.h"
 
 @interface Tree (internal)
 - (BOOL)readHeader:(BufferedInputStream *)is error:(NSError **)error;
 @end
 
 @implementation Tree
-@synthesize xattrsSHA1, xattrsSize, aclSHA1, uid, gid, mode, mtime_sec, mtime_nsec, flags, finderFlags, extendedFinderFlags, treeVersion, st_rdev;
+@synthesize xattrsAreCompressed, xattrsBlobKey, xattrsSize, aclIsCompressed, aclBlobKey, uid, gid, mode, mtime_sec, mtime_nsec, flags, finderFlags, extendedFinderFlags, st_dev, treeVersion, st_rdev;
-@synthesize ctime_sec, ctime_nsec, createTime_sec, createTime_nsec, st_nlink, st_ino;
+@synthesize ctime_sec, ctime_nsec, createTime_sec, createTime_nsec, st_nlink, st_ino, st_blocks, st_blksize;
+@synthesize aggregateSizeOnDisk;
 
 + (NSString *)errorDomain {
     return @"TreeErrorDomain";
 }
+- (id)init {
+    if (self = [super init]) {
+        nodes = [[NSMutableDictionary alloc] init];
+    }
+    return self;
+}
 - (id)initWithBufferedInputStream:(BufferedInputStream *)is error:(NSError **)error {
 	if (self = [super init]) {
         if (![self readHeader:is error:error]) {
             [self release];
             return nil;
         }
+        if (treeVersion >= 12) {
+            if (![BooleanIO read:&xattrsAreCompressed from:is error:error]
+                || ![BooleanIO read:&aclIsCompressed from:is error:error]) {
+                [self release];
+                return nil;
+            }
+        }
+        
+        NSString *xattrsSHA1 = nil;
+        BOOL xattrsStretchedKey = NO;
+        NSString *aclSHA1 = nil;
+        BOOL aclStretchedKey = NO;
         BOOL ret = [StringIO read:&xattrsSHA1 from:is error:error]
+        && (treeVersion < 14 || [BooleanIO read:&xattrsStretchedKey from:is error:error])
         && [IntegerIO readUInt64:&xattrsSize from:is error:error]
         &&[StringIO read:&aclSHA1 from:is error:error]
+        && (treeVersion < 14 || [BooleanIO read:&aclStretchedKey from:is error:error])
         && [IntegerIO readInt32:&uid from:is error:error]
         && [IntegerIO readInt32:&gid from:is error:error]
         && [IntegerIO readInt32:&mode from:is error:error]
@@ -78,12 +79,21 @@
         && [IntegerIO readInt64:&ctime_nsec from:is error:error]
         && [IntegerIO readInt64:&st_blocks from:is error:error]
         && [IntegerIO readUInt32:&st_blksize from:is error:error];
-        [xattrsSHA1 retain];
-        [aclSHA1 retain];
-        
         if (!ret) {
             goto initError;
         }
+        if (xattrsSHA1 != nil) {
+            xattrsBlobKey = [[BlobKey alloc] initWithSHA1:xattrsSHA1 stretchEncryptionKey:xattrsStretchedKey];
+        }
+        if (aclSHA1 != nil) {
+            aclBlobKey = [[BlobKey alloc] initWithSHA1:aclSHA1 stretchEncryptionKey:aclStretchedKey];
+        }
+        
+        if (treeVersion >= 11) {
+            if (![IntegerIO readUInt64:&aggregateSizeOnDisk from:is error:error]) {
+                goto initError;
+            }
+        }
         
         unsigned int nodeCount;
         if (![IntegerIO readUInt32:&nodeCount from:is error:error]) {
@@ -111,8 +121,8 @@ initDone:
 	return self;
 }
 - (void)dealloc {
-    [xattrsSHA1 release];
+    [xattrsBlobKey release];
-    [aclSHA1 release];
+    [aclBlobKey release];
 	[nodes release];
 	[super dealloc];
 }
@@ -125,14 +135,18 @@ initDone:
 - (BOOL)containsNodeNamed:(NSString *)name {
 	return [nodes objectForKey:name] != nil;
 }
+- (NSDictionary *)nodes {
+    return nodes;
+}
 - (Blob *)toBlob {
     NSMutableData *data = [[NSMutableData alloc] init];
-    char header[TREE_HEADER_LENGTH + 1];
+    [BooleanIO write:xattrsAreCompressed to:data];
-    sprintf(header, "TreeV%03d", CURRENT_TREE_VERSION);
+    [BooleanIO write:aclIsCompressed to:data];
-    [data appendBytes:header length:TREE_HEADER_LENGTH];
+    [StringIO write:[xattrsBlobKey sha1] to:data];
-    [StringIO write:xattrsSHA1 to:data];
+    [BooleanIO write:[xattrsBlobKey stretchEncryptionKey] to:data];
     [IntegerIO writeUInt64:xattrsSize to:data];
-    [StringIO write:aclSHA1 to:data];
+    [StringIO write:[aclBlobKey sha1] to:data];
+    [BooleanIO write:[aclBlobKey stretchEncryptionKey] to:data];
     [IntegerIO writeInt32:uid to:data];
     [IntegerIO writeInt32:gid to:data];
     [IntegerIO writeInt32:mode to:data];
@@ -149,6 +163,7 @@ initDone:
     [IntegerIO writeInt64:ctime_nsec to:data];
     [IntegerIO writeInt64:st_blocks to:data];
     [IntegerIO writeUInt32:st_blksize to:data];
+    [IntegerIO writeUInt64:aggregateSizeOnDisk to:data];
     
     [IntegerIO writeUInt32:(uint32_t)[nodes count] to:data];
     NSMutableArray *nodeNames = [NSMutableArray arrayWithArray:[nodes allKeys]];
@@ -158,19 +173,66 @@ initDone:
         Node *node = [nodes objectForKey:nodeName];
         [node writeToData:data];
     }
-    Blob *ret =[[[Blob alloc] initWithData:data mimeType:@"binary/octet-stream" downloadName:@"Tree"] autorelease];
+    
+    char header[TREE_HEADER_LENGTH + 1];
+    sprintf(header, "TreeV%03d", CURRENT_TREE_VERSION);
+    NSMutableData *completeData = [[NSMutableData alloc] init];
+    [completeData appendBytes:header length:TREE_HEADER_LENGTH];
+
+    [completeData appendBytes:[data bytes] length:[data length]];
+    
+    Blob *ret =[[[Blob alloc] initWithData:completeData mimeType:@"binary/octet-stream" downloadName:@"Tree" dataDescription:@"tree"] autorelease];
+    [completeData release];
     [data release];
     return ret;
 }
+
+#pragma mark NSObject
+- (BOOL)isEqual:(id)object {
+    if (![object isKindOfClass:[Tree class]]) {
+        return NO;
+    }
+    Tree *other = (Tree *)object;
+    return treeVersion == [other treeVersion] 
+    && xattrsAreCompressed == [other xattrsAreCompressed]
+    && [NSObject equalObjects:xattrsBlobKey and:[other xattrsBlobKey]]
+    && xattrsSize == [other xattrsSize]
+    && aclIsCompressed == [other aclIsCompressed]
+    && [NSObject equalObjects:aclBlobKey and:[other aclBlobKey]]
+    && uid == [other uid]
+    && gid == [other gid]
+    && mode == [other mode]
+    && mtime_sec == [other mtime_sec]
+    && mtime_nsec == [other mtime_nsec]
+    && flags == [other flags]
+    && finderFlags == [other finderFlags]
+    && extendedFinderFlags == [other extendedFinderFlags]
+    && st_dev == [other st_dev]
+    && st_ino == [other st_ino]
+    && st_nlink == [other st_nlink]
+    && st_rdev == [other st_rdev]
+    && ctime_sec == [other ctime_sec]
+    && ctime_nsec == [other ctime_nsec]
+    && createTime_sec == [other createTime_sec]
+    && createTime_nsec == [other createTime_nsec]
+    && st_blocks == [other st_blocks]
+    && st_blksize == [other st_blksize]
+    && aggregateSizeOnDisk == [other aggregateSizeOnDisk]
+    && [nodes isEqual:[other nodes]];
+}
+- (NSUInteger)hash {
+    return (NSUInteger)treeVersion + [nodes hash];
+}
 @end
 
 @implementation Tree (internal)
 - (BOOL)readHeader:(BufferedInputStream *)is error:(NSError **)error {
-    NSData *headerData = [is readExactly:TREE_HEADER_LENGTH error:error];
+    BOOL ret = NO;
-    if (headerData == nil) {
+    unsigned char *buf = (unsigned char *)malloc(TREE_HEADER_LENGTH);
-        return NO;
+    if (![is readExactly:TREE_HEADER_LENGTH into:buf error:error]) {
+        goto readHeader_error;
     }
-    NSString *header = [[[NSString alloc] initWithData:headerData encoding:NSUTF8StringEncoding] autorelease];
+    NSString *header = [[[NSString alloc] initWithBytes:buf length:TREE_HEADER_LENGTH encoding:NSASCIIStringEncoding] autorelease];
     NSRange versionRange = [header rangeOfRegex:@"^TreeV(\\d{3})$" capture:1];
     treeVersion = 0;
     if (versionRange.location != NSNotFound) {
@@ -179,10 +241,17 @@ initDone:
         treeVersion = [number intValue];
         [nf release];
     }
-    if (treeVersion != CURRENT_TREE_VERSION) {
+    if (treeVersion < 10) {
         SETNSERROR([Tree errorDomain], ERROR_INVALID_OBJECT_VERSION, @"invalid Tree header: %@", header);
-        return NO;
+        goto readHeader_error;
     }
-    return YES;
+    if (treeVersion == 13) {
+        SETNSERROR([Tree errorDomain], ERROR_INVALID_OBJECT_VERSION, @"invalid Tree version 13");
+        goto readHeader_error;
+    }
+    ret = YES;
+readHeader_error:
+    free(buf);
+    return ret;
 }
 @end

UserAndComputer.h

@@ -13,7 +13,9 @@
     NSString *userName;
     NSString *computerName;
 }
+- (id)init;
 - (id)initWithXMLData:(NSData *)theXMLData error:(NSError **)error;
+- (id)initWithUserName:(NSString *)theUserName computerName:(NSString *)theComputerName;
 - (NSString *)userName;
 - (NSString *)computerName;
 - (NSData *)toXMLData;

UserAndComputer.m

@@ -7,9 +7,17 @@
 //
 
 #import "UserAndComputer.h"
+#import "Computer.h"
 #import "DictNode.h"
 
 @implementation UserAndComputer
+- (id)init {
+    if (self = [super init]) {
+        userName = [NSUserName() copy];
+        computerName = [[Computer name] copy];
+    }
+    return self;
+}
 - (id)initWithXMLData:(NSData *)theXMLData error:(NSError **)error {
     if (self = [super init]) {
         DictNode *plist = [DictNode dictNodeWithXMLData:theXMLData error:error];
@@ -22,6 +30,13 @@
     }
     return self;
 }
+- (id)initWithUserName:(NSString *)theUserName computerName:(NSString *)theComputerName {
+    if (self = [super init]) {
+        userName = [theUserName retain];
+        computerName = [theComputerName retain];
+    }
+    return self;
+}
 - (void)dealloc {
     [userName release];
     [computerName release];

UserLibrary_Arq.h

@@ -0,0 +1,15 @@
+//
+//  UserLibrary.h
+//  Backup
+//
+//  Created by Stefan Reitshamer on 8/18/09.
+//  Copyright 2009 PhotoMinds LLC. All rights reserved.
+//
+
+#import <Cocoa/Cocoa.h>
+#import "UserLibrary.h"
+
+@interface UserLibrary (Arq)
++ (NSString *)arqUserLibraryPath;
++ (NSString *)arqCachePath;
+@end

UserLibrary_Arq.m

@@ -0,0 +1,19 @@
+//
+//  UserLibrary.m
+//  Backup
+//
+//  Created by Stefan Reitshamer on 8/18/09.
+//  Copyright 2009 PhotoMinds LLC. All rights reserved.
+//
+
+#import "UserLibrary_Arq.h"
+
+
+@implementation UserLibrary (Arq)
++ (NSString *)arqUserLibraryPath {
+	return [NSHomeDirectory() stringByAppendingString:@"/Library/Arq"];
+}
++ (NSString *)arqCachePath {
+    return [NSString stringWithFormat:@"%@/Cache.noindex", [UserLibrary arqUserLibraryPath]];
+}
+@end

XAttrSet.h

@@ -1,34 +1,10 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  XAttrSet.h
- 
+//  Backup
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 4/27/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 PhotoMinds LLC. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #import <Cocoa/Cocoa.h>
 #import "Blob.h"
@@ -36,10 +12,11 @@
 
 @interface XAttrSet : NSObject {
     NSMutableDictionary *xattrs;
+    NSString *path;
 }
 - (id)initWithPath:(NSString *)thePath error:(NSError **)error;
 - (id)initWithBufferedInputStream:(BufferedInputStream *)is error:(NSError **)error;
-- (Blob *)toBlob;
+- (NSData *)toData;
 - (NSUInteger)count;
 - (unsigned long long)dataLength;
 - (NSArray *)names;

XAttrSet.m

@@ -1,34 +1,10 @@
-/*
+//
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+//  XAttrSet.m
- 
+//  Backup
- All rights reserved.
+//
- 
+//  Created by Stefan Reitshamer on 4/27/09.
- Redistribution and use in source and binary forms, with or without
+//  Copyright 2009 PhotoMinds LLC. All rights reserved.
- modification, are permitted provided that the following conditions are met:
+//
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
 
 #include <sys/stat.h>
 #include <sys/xattr.h>
@@ -38,11 +14,12 @@
 #import "IntegerIO.h"
 #import "Blob.h"
 #import "DataInputStream.h"
-#import "BufferedInputStream.h"
 #import "SetNSError.h"
 #import "NSErrorCodes.h"
 #import "Streams.h"
 #import "NSError_extra.h"
+#import "BufferedInputStream.h"
+#import "NSData-Gzip.h"
 
 #define HEADER_LENGTH (12)
 
@@ -64,9 +41,10 @@
                     *error = myError;
                 }
                 [self release];
-                self = nil;
+                return nil;
             }
         }
+        path = [thePath retain];
     }
     return self;
 }
@@ -82,20 +60,19 @@
 }
 - (void)dealloc {
     [xattrs release];
+    [path release];
     [super dealloc];
 }
-- (Blob *)toBlob {
+- (NSData *)toData {
-    NSMutableData *data = [[NSMutableData alloc] init];
+    NSMutableData *mutableData = [[[NSMutableData alloc] init] autorelease];
-    [data appendBytes:"XAttrSetV002" length:HEADER_LENGTH];
+    [mutableData appendBytes:"XAttrSetV002" length:HEADER_LENGTH];
     uint64_t count = (uint64_t)[xattrs count];
-    [IntegerIO writeUInt64:count to:data];
+    [IntegerIO writeUInt64:count to:mutableData];
     for (NSString *name in [xattrs allKeys]) {
-        [StringIO write:name to:data];
+        [StringIO write:name to:mutableData];
-        [DataIO write:[xattrs objectForKey:name] to:data];
+        [DataIO write:[xattrs objectForKey:name] to:mutableData];
     }
-    Blob *ret = [[[Blob alloc] initWithData:data mimeType:@"binary/octet-stream" downloadName:@"xattrset"] autorelease];    
+    return mutableData;
-    [data release];
-    return ret;
 }
 - (NSUInteger)count {
     return [xattrs count];
@@ -111,15 +88,17 @@
 - (NSArray *)names {
     return [xattrs allKeys];
 }
-- (BOOL)applyToFile:(NSString *)path error:(NSError **)error {
+- (BOOL)applyToFile:(NSString *)thePath error:(NSError **)error {
-    XAttrSet *current = [[[XAttrSet alloc] initWithPath:path error:error] autorelease];
+    XAttrSet *current = [[[XAttrSet alloc] initWithPath:thePath error:error] autorelease];
     if (!current) {
         return NO;
     }
-    const char *pathChars = [path fileSystemRepresentation];
+    const char *pathChars = [thePath fileSystemRepresentation];
     for (NSString *name in [current names]) {
         if (removexattr(pathChars, [name UTF8String], XATTR_NOFOLLOW) == -1) {
-            SETNSERROR(@"UnixErrorDomain", errno, @"removexattr: %s", strerror(errno));
+            int errnum = errno;
+            HSLogError(@"removexattr(%@, %@) error %d: %s", thePath, name, errnum, strerror(errnum));
+            SETNSERROR(@"UnixErrorDomain", errnum, @"failed to remove extended attribute %@ from %@: %s", name, thePath, strerror(errnum));
             return NO;
         }
     }
@@ -131,7 +110,9 @@
                      [value length],
                      0,
                      XATTR_NOFOLLOW) == -1) {
-            SETNSERROR(@"UnixErrorDomain", errno, @"setxattr: %s", strerror(errno));
+            int errnum = errno;
+            HSLogError(@"setxattr(%@, %@) error %d: %s", thePath, key, errnum, strerror(errnum));
+            SETNSERROR(@"UnixErrorDomain", errnum, @"failed to set extended attribute %@ on %@: %s", key, thePath, strerror(errnum));
             return NO;
         }
     }
@@ -143,37 +124,47 @@
 - (BOOL)loadFromPath:(NSString *)thePath error:(NSError **)error {
     struct stat st;
     if (lstat([thePath fileSystemRepresentation], &st) == -1) {
-        SETNSERROR(@"UnixErrorDomain", errno, @"lstat(%@): %s", thePath, strerror(errno));
+        int errnum = errno;
+        HSLogError(@"lstat(%@) error %d: %s", thePath, errnum, strerror(errnum));
+        SETNSERROR(@"UnixErrorDomain", errnum, @"%@: %s", thePath, strerror(errnum));
         return NO;
     }
     if (S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)) {
-        const char *path = [thePath fileSystemRepresentation];
+        const char *cpath = [thePath fileSystemRepresentation];
-        ssize_t xattrsize = listxattr(path, NULL, 0, XATTR_NOFOLLOW);
+        ssize_t xattrsize = listxattr(cpath, NULL, 0, XATTR_NOFOLLOW);
         if (xattrsize == -1) {
-            SETNSERROR(@"UnixErrorDomain", errno, @"%s", strerror(errno));
+            int errnum = errno;
+            HSLogError(@"listxattr(%@) error %d: %s", thePath, errnum, strerror(errnum));
+            SETNSERROR(@"UnixErrorDomain", errnum, @"failed to list extended attributes of %@: %s", thePath, strerror(errnum));
             return NO;
         } 
         if (xattrsize > 0) {
             char *xattrbuf = (char *)malloc(xattrsize);
-            xattrsize = listxattr(path, xattrbuf, xattrsize, XATTR_NOFOLLOW);
+            xattrsize = listxattr(cpath, xattrbuf, xattrsize, XATTR_NOFOLLOW);
             if (xattrsize == -1) {
-                SETNSERROR(@"UnixErrorDomain", errno, @"%s", strerror(errno));
+                int errnum = errno;
+                HSLogError(@"listxattr(%@) error %d: %s", thePath, errnum, strerror(errnum));
+                SETNSERROR(@"UnixErrorDomain", errnum, @"failed to list extended attributes of %@: %s", thePath, strerror(errnum));
                 free(xattrbuf);
                 return NO;
             }
             for (char *name = xattrbuf; name < (xattrbuf + xattrsize); name += strlen(name) + 1) {
                 NSString *theName = [NSString stringWithUTF8String:name];
-                ssize_t valuesize = getxattr(path, name, NULL, 0, 0, XATTR_NOFOLLOW);
+                ssize_t valuesize = getxattr(cpath, name, NULL, 0, 0, XATTR_NOFOLLOW);
                 NSData *xattrData = nil;
                 if (valuesize == -1) {
-                    SETNSERROR(@"UnixErrorDomain", errno, @"Error reading extended attribute %s: %s", name, strerror(errno));
+                    int errnum = errno;
+                    HSLogError(@"getxattr(%s, %s) error %d: %s", cpath, name, errnum, strerror(errnum));
+                    SETNSERROR(@"UnixErrorDomain", errnum, @"failed to read extended attribute %s of %@: %s", name, thePath, strerror(errnum));
                     free(xattrbuf);
                     return NO;
                 }
                 if (valuesize > 0) {
                     void *value = malloc(valuesize);
-                    if (getxattr(path, name, value, valuesize, 0, XATTR_NOFOLLOW) == -1) {
+                    if (getxattr(cpath, name, value, valuesize, 0, XATTR_NOFOLLOW) == -1) {
-                        SETNSERROR(@"UnixErrorDomain", errno, @"getxattr: %s", strerror(errno));
+                        int errnum = errno;
+                        HSLogError(@"getxattr(%s, %s) error %d: %s", cpath, name, errnum, strerror(errnum));
+                        SETNSERROR(@"UnixErrorDomain", errnum, @"failed to read extended attribute %s of %@: %s", name, thePath, strerror(errnum));
                         free(value);
                         free(xattrbuf);
                         return NO;
@@ -191,29 +182,33 @@
     return YES;
 }
 - (BOOL)loadFromInputStream:(BufferedInputStream *)is error:(NSError **)error {
-    NSData *headerData = [is readExactly:HEADER_LENGTH error:error];
+    BOOL ret = NO;
-    if (headerData == nil) {
+    unsigned char *buf = (unsigned char *)malloc(HEADER_LENGTH);
-        return NO;
+    if (![is readExactly:HEADER_LENGTH into:buf error:error]) {
+        goto load_error;
     }
-    if (strncmp((const char *)[headerData bytes], "XAttrSetV002", HEADER_LENGTH)) {
+    if (strncmp((const char *)buf, "XAttrSetV002", HEADER_LENGTH)) {
         SETNSERROR(@"XAttrSetErrorDomain", ERROR_INVALID_OBJECT_VERSION, @"invalid XAttrSet header");
-        return NO;
+        goto load_error;
     }
     uint64_t count;
     if (![IntegerIO readUInt64:&count from:is error:error]) {
-        return NO;
+        goto load_error;
     }
     for (uint64_t i = 0; i < count; i++) {
         NSString *name;
         if (![StringIO read:&name from:is error:error]) {
-            return NO;
+            goto load_error;
         }
         NSData *value;
         if (![DataIO read:&value from:is error:error]) {
-            return NO;
+            goto load_error;
         }
         [xattrs setObject:value forKey:name];
     }
-    return YES;
+    ret = YES;
+load_error:
+    free(buf);
+    return ret;
 }
 @end

arq_restore.m

@@ -33,7 +33,7 @@
 #include <libgen.h>
 #import <Foundation/Foundation.h>
 #import "ArqRestoreCommand.h"
-#import "ArqFolder.h"
+
 
 static void printUsage(const char *exeName) {
 	fprintf(stderr, "Usage:\n");

arq_restore.xcodeproj/project.pbxproj

@@ -9,8 +9,6 @@
 /* Begin PBXBuildFile section */
 		8DD76F9C0486AA7600D96B5E /* Foundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 08FB779EFE84155DC02AAC07 /* Foundation.framework */; };
 		F805B54D1160D3E6007EC01E /* ArqRestoreCommand.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B54C1160D3E6007EC01E /* ArqRestoreCommand.m */; };
-		F805B7211160D9C2007EC01E /* HSLog.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7201160D9C2007EC01E /* HSLog.m */; };
-		F805B7301160DBE9007EC01E /* ArqFolder.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B72F1160DBE9007EC01E /* ArqFolder.m */; };
 		F805B7531160DCFE007EC01E /* ArrayNode.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7421160DCFE007EC01E /* ArrayNode.m */; };
 		F805B7541160DCFE007EC01E /* BooleanNode.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7441160DCFE007EC01E /* BooleanNode.m */; };
 		F805B7551160DCFE007EC01E /* DictNode.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7461160DCFE007EC01E /* DictNode.m */; };
@@ -19,22 +17,16 @@
 		F805B7581160DCFE007EC01E /* StringNode.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B74E1160DCFE007EC01E /* StringNode.m */; };
 		F805B7591160DCFE007EC01E /* XMLPListReader.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7501160DCFE007EC01E /* XMLPListReader.m */; };
 		F805B75A1160DCFE007EC01E /* XMLPListWriter.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7521160DCFE007EC01E /* XMLPListWriter.m */; };
-		F805B7821160DD60007EC01E /* HTTPConnection_S3.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7691160DD60007EC01E /* HTTPConnection_S3.m */; };
 		F805B7831160DD60007EC01E /* NSError_S3.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B76B1160DD60007EC01E /* NSError_S3.m */; };
 		F805B7841160DD60007EC01E /* PathReceiver.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B76D1160DD60007EC01E /* PathReceiver.m */; };
-		F805B7851160DD60007EC01E /* S3AuthorizationParameters.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B76F1160DD60007EC01E /* S3AuthorizationParameters.m */; };
 		F805B7861160DD60007EC01E /* S3AuthorizationProvider.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7711160DD60007EC01E /* S3AuthorizationProvider.m */; };
 		F805B7871160DD60007EC01E /* S3Lister.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7731160DD60007EC01E /* S3Lister.m */; };
 		F805B7881160DD60007EC01E /* S3ObjectMetadata.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7751160DD60007EC01E /* S3ObjectMetadata.m */; };
 		F805B7891160DD60007EC01E /* S3ObjectReceiver.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7771160DD60007EC01E /* S3ObjectReceiver.m */; };
 		F805B78B1160DD60007EC01E /* S3Request.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B77C1160DD60007EC01E /* S3Request.m */; };
 		F805B78C1160DD60007EC01E /* S3Service.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B77E1160DD60007EC01E /* S3Service.m */; };
-		F805B78D1160DD60007EC01E /* S3Signature.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7801160DD60007EC01E /* S3Signature.m */; };
 		F805B7A91160DEF2007EC01E /* RegexKitLite.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7A81160DEF2007EC01E /* RegexKitLite.m */; };
 		F805B7BA1160E3AF007EC01E /* Blob.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7B91160E3AF007EC01E /* Blob.m */; };
-		F805B7D11160E445007EC01E /* HTTPConnection.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7CC1160E445007EC01E /* HTTPConnection.m */; };
-		F805B7D21160E445007EC01E /* HTTPRequest.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7CE1160E445007EC01E /* HTTPRequest.m */; };
-		F805B7D31160E445007EC01E /* HTTPResponse.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7D01160E445007EC01E /* HTTPResponse.m */; };
 		F805B7D81160E456007EC01E /* BlobACL.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7D71160E456007EC01E /* BlobACL.m */; };
 		F805B7E11160E48B007EC01E /* ServerBlob.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7E01160E48B007EC01E /* ServerBlob.m */; };
 		F805B7FB1160E73D007EC01E /* RFC2616DateFormatter.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7FA1160E73D007EC01E /* RFC2616DateFormatter.m */; };
@@ -48,9 +40,6 @@
 		F805B8321160E878007EC01E /* Writer.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B8311160E878007EC01E /* Writer.m */; };
 		F805B83B1160E8DD007EC01E /* NSData-InputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B83A1160E8DD007EC01E /* NSData-InputStream.m */; };
 		F805B8421160E90F007EC01E /* Streams.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B8411160E90F007EC01E /* Streams.m */; };
-		F805B85A1160E9C9007EC01E /* CFStreamInputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B8571160E9C9007EC01E /* CFStreamInputStream.m */; };
-		F805B85B1160E9C9007EC01E /* CFStreamOutputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B8591160E9C9007EC01E /* CFStreamOutputStream.m */; };
-		F805B8601160E9F0007EC01E /* DNS_SDErrors.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B85F1160E9F0007EC01E /* DNS_SDErrors.m */; };
 		F805B8651160EA15007EC01E /* NSXMLNode_extra.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B8641160EA15007EC01E /* NSXMLNode_extra.m */; };
 		F805B86A1160EA83007EC01E /* NSData-Base64Extensions.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B8691160EA83007EC01E /* NSData-Base64Extensions.m */; };
 		F805B86F1160EAC1007EC01E /* DataInputStreamFactory.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B86E1160EAC1007EC01E /* DataInputStreamFactory.m */; };
@@ -62,8 +51,6 @@
 		F805B8C21160EC41007EC01E /* Security.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = F805B8C11160EC41007EC01E /* Security.framework */; };
 		F805B8CE1160ECD7007EC01E /* CoreServices.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = F805B8CD1160ECD7007EC01E /* CoreServices.framework */; };
 		F83C1A7411CA7C170001958F /* ArqRestoreCommand.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B54C1160D3E6007EC01E /* ArqRestoreCommand.m */; };
-		F83C1A7511CA7C170001958F /* HSLog.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7201160D9C2007EC01E /* HSLog.m */; };
-		F83C1A7611CA7C170001958F /* ArqFolder.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B72F1160DBE9007EC01E /* ArqFolder.m */; };
 		F83C1A7711CA7C170001958F /* ArrayNode.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7421160DCFE007EC01E /* ArrayNode.m */; };
 		F83C1A7811CA7C170001958F /* BooleanNode.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7441160DCFE007EC01E /* BooleanNode.m */; };
 		F83C1A7911CA7C170001958F /* DictNode.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7461160DCFE007EC01E /* DictNode.m */; };
@@ -72,22 +59,16 @@
 		F83C1A7C11CA7C170001958F /* StringNode.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B74E1160DCFE007EC01E /* StringNode.m */; };
 		F83C1A7D11CA7C170001958F /* XMLPListReader.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7501160DCFE007EC01E /* XMLPListReader.m */; };
 		F83C1A7E11CA7C170001958F /* XMLPListWriter.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7521160DCFE007EC01E /* XMLPListWriter.m */; };
-		F83C1A7F11CA7C170001958F /* HTTPConnection_S3.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7691160DD60007EC01E /* HTTPConnection_S3.m */; };
 		F83C1A8011CA7C170001958F /* NSError_S3.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B76B1160DD60007EC01E /* NSError_S3.m */; };
 		F83C1A8111CA7C170001958F /* PathReceiver.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B76D1160DD60007EC01E /* PathReceiver.m */; };
-		F83C1A8211CA7C170001958F /* S3AuthorizationParameters.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B76F1160DD60007EC01E /* S3AuthorizationParameters.m */; };
 		F83C1A8311CA7C170001958F /* S3AuthorizationProvider.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7711160DD60007EC01E /* S3AuthorizationProvider.m */; };
 		F83C1A8411CA7C170001958F /* S3Lister.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7731160DD60007EC01E /* S3Lister.m */; };
 		F83C1A8511CA7C170001958F /* S3ObjectMetadata.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7751160DD60007EC01E /* S3ObjectMetadata.m */; };
 		F83C1A8611CA7C170001958F /* S3ObjectReceiver.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7771160DD60007EC01E /* S3ObjectReceiver.m */; };
 		F83C1A8711CA7C170001958F /* S3Request.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B77C1160DD60007EC01E /* S3Request.m */; };
 		F83C1A8811CA7C170001958F /* S3Service.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B77E1160DD60007EC01E /* S3Service.m */; };
-		F83C1A8911CA7C170001958F /* S3Signature.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7801160DD60007EC01E /* S3Signature.m */; };
 		F83C1A8A11CA7C170001958F /* RegexKitLite.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7A81160DEF2007EC01E /* RegexKitLite.m */; };
 		F83C1A8B11CA7C170001958F /* Blob.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7B91160E3AF007EC01E /* Blob.m */; };
-		F83C1A8C11CA7C170001958F /* HTTPConnection.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7CC1160E445007EC01E /* HTTPConnection.m */; };
-		F83C1A8D11CA7C170001958F /* HTTPRequest.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7CE1160E445007EC01E /* HTTPRequest.m */; };
-		F83C1A8E11CA7C170001958F /* HTTPResponse.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7D01160E445007EC01E /* HTTPResponse.m */; };
 		F83C1A8F11CA7C170001958F /* BlobACL.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7D71160E456007EC01E /* BlobACL.m */; };
 		F83C1A9011CA7C170001958F /* ServerBlob.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7E01160E48B007EC01E /* ServerBlob.m */; };
 		F83C1A9111CA7C170001958F /* RFC2616DateFormatter.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B7FA1160E73D007EC01E /* RFC2616DateFormatter.m */; };
@@ -101,15 +82,11 @@
 		F83C1A9911CA7C170001958F /* Writer.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B8311160E878007EC01E /* Writer.m */; };
 		F83C1A9A11CA7C170001958F /* NSData-InputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B83A1160E8DD007EC01E /* NSData-InputStream.m */; };
 		F83C1A9C11CA7C170001958F /* Streams.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B8411160E90F007EC01E /* Streams.m */; };
-		F83C1A9E11CA7C170001958F /* CFStreamInputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B8571160E9C9007EC01E /* CFStreamInputStream.m */; };
-		F83C1A9F11CA7C170001958F /* CFStreamOutputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B8591160E9C9007EC01E /* CFStreamOutputStream.m */; };
-		F83C1AA011CA7C170001958F /* DNS_SDErrors.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B85F1160E9F0007EC01E /* DNS_SDErrors.m */; };
 		F83C1AA111CA7C170001958F /* NSXMLNode_extra.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B8641160EA15007EC01E /* NSXMLNode_extra.m */; };
 		F83C1AA211CA7C170001958F /* NSData-Base64Extensions.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B8691160EA83007EC01E /* NSData-Base64Extensions.m */; };
 		F83C1AA311CA7C170001958F /* DataInputStreamFactory.m in Sources */ = {isa = PBXBuildFile; fileRef = F805B86E1160EAC1007EC01E /* DataInputStreamFactory.m */; };
 		F83C1AA411CA7C170001958F /* Restorer.m in Sources */ = {isa = PBXBuildFile; fileRef = F8D677FF1160F26A00CC270E /* Restorer.m */; };
 		F83C1AA711CA7C170001958F /* SHA1Hash.m in Sources */ = {isa = PBXBuildFile; fileRef = F8D6781C1160F4FD00CC270E /* SHA1Hash.m */; };
-		F83C1AAA11CA7C170001958F /* ArqUserLibrary.m in Sources */ = {isa = PBXBuildFile; fileRef = F8D678321160F62E00CC270E /* ArqUserLibrary.m */; };
 		F83C1AAB11CA7C170001958F /* PackIndexEntry.m in Sources */ = {isa = PBXBuildFile; fileRef = F8D6783B1160F70100CC270E /* PackIndexEntry.m */; };
 		F83C1AAC11CA7C170001958F /* DiskPack.m in Sources */ = {isa = PBXBuildFile; fileRef = F8D678421160F74A00CC270E /* DiskPack.m */; };
 		F83C1AAD11CA7C170001958F /* DiskPackIndex.m in Sources */ = {isa = PBXBuildFile; fileRef = F8D678441160F74A00CC270E /* DiskPackIndex.m */; };
@@ -150,22 +127,51 @@
 		F83C1AD111CA7C170001958F /* Security.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = F805B8C11160EC41007EC01E /* Security.framework */; };
 		F83C1AD211CA7C170001958F /* CoreServices.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = F805B8CD1160ECD7007EC01E /* CoreServices.framework */; };
 		F83C1AE311CA7C7C0001958F /* arq_restore.m in Sources */ = {isa = PBXBuildFile; fileRef = 08FB7796FE84155DC02AAC07 /* arq_restore.m */; };
-		F83C1D0A11CA929D0001958F /* BucketVerifier.m in Sources */ = {isa = PBXBuildFile; fileRef = F83C1D0911CA929D0001958F /* BucketVerifier.m */; };
 		F83C1D1D11CA95AF0001958F /* BucketVerifier.m in Sources */ = {isa = PBXBuildFile; fileRef = F83C1D0911CA929D0001958F /* BucketVerifier.m */; };
 		F8987235121EB68900F07D76 /* BinaryPListReader.m in Sources */ = {isa = PBXBuildFile; fileRef = F8987232121EB68900F07D76 /* BinaryPListReader.m */; };
 		F8987236121EB68900F07D76 /* BinaryPListWriter.m in Sources */ = {isa = PBXBuildFile; fileRef = F8987234121EB68900F07D76 /* BinaryPListWriter.m */; };
-		F8987527121EB89100F07D76 /* CFStreamPair.m in Sources */ = {isa = PBXBuildFile; fileRef = F8987526121EB89100F07D76 /* CFStreamPair.m */; };
-		F8987532121EB94000F07D76 /* StreamPairFactory.m in Sources */ = {isa = PBXBuildFile; fileRef = F8987531121EB94000F07D76 /* StreamPairFactory.m */; };
 		F8987560121EBD9600F07D76 /* BufferedInputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = F898755F121EBD9600F07D76 /* BufferedInputStream.m */; };
 		F8987588121EBDF300F07D76 /* BinaryPListReader.m in Sources */ = {isa = PBXBuildFile; fileRef = F8987232121EB68900F07D76 /* BinaryPListReader.m */; };
 		F8987589121EBDF400F07D76 /* BinaryPListWriter.m in Sources */ = {isa = PBXBuildFile; fileRef = F8987234121EB68900F07D76 /* BinaryPListWriter.m */; };
 		F898758A121EBDFA00F07D76 /* BufferedInputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = F898755F121EBD9600F07D76 /* BufferedInputStream.m */; };
-		F898758B121EBDFD00F07D76 /* CFStreamPair.m in Sources */ = {isa = PBXBuildFile; fileRef = F8987526121EB89100F07D76 /* CFStreamPair.m */; };
 		F898758C121EBE0600F07D76 /* UserAndComputer.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F4D67D121DA542002D09C1 /* UserAndComputer.m */; };
-		F898758D121EBE0800F07D76 /* StreamPairFactory.m in Sources */ = {isa = PBXBuildFile; fileRef = F8987531121EB94000F07D76 /* StreamPairFactory.m */; };
+		F89A1EEC13FAC4E30071D321 /* URLConnection.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1EEB13FAC4E30071D321 /* URLConnection.m */; };
+		F89A1EED13FAC4E30071D321 /* URLConnection.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1EEB13FAC4E30071D321 /* URLConnection.m */; };
+		F89A1EFB13FAC5970071D321 /* EncryptedInputStreamFactory.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1EFA13FAC5970071D321 /* EncryptedInputStreamFactory.m */; };
+		F89A1EFC13FAC5970071D321 /* EncryptedInputStreamFactory.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1EFA13FAC5970071D321 /* EncryptedInputStreamFactory.m */; };
+		F89A1F2B13FAC6700071D321 /* UserLibrary.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F2A13FAC6700071D321 /* UserLibrary.m */; };
+		F89A1F2C13FAC6700071D321 /* UserLibrary.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F2A13FAC6700071D321 /* UserLibrary.m */; };
+		F89A1F3E13FAC6820071D321 /* UserLibrary_Arq.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F3D13FAC6820071D321 /* UserLibrary_Arq.m */; };
+		F89A1F3F13FAC6820071D321 /* UserLibrary_Arq.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F3D13FAC6820071D321 /* UserLibrary_Arq.m */; };
+		F89A1F4513FAC6C40071D321 /* BlobKey.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F4413FAC6C40071D321 /* BlobKey.m */; };
+		F89A1F4613FAC6C40071D321 /* BlobKey.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F4413FAC6C40071D321 /* BlobKey.m */; };
+		F89A1F4D13FAC73D0071D321 /* Computer.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F4C13FAC73D0071D321 /* Computer.m */; };
+		F89A1F4E13FAC73D0071D321 /* Computer.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F4C13FAC73D0071D321 /* Computer.m */; };
+		F89A1F5313FAC78F0071D321 /* BufferedOutputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F5213FAC78F0071D321 /* BufferedOutputStream.m */; };
+		F89A1F5413FAC78F0071D321 /* BufferedOutputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F5213FAC78F0071D321 /* BufferedOutputStream.m */; };
+		F89A1F5B13FAC7D50071D321 /* Encryption.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F5A13FAC7D50071D321 /* Encryption.m */; };
+		F89A1F5C13FAC7D50071D321 /* Encryption.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F5A13FAC7D50071D321 /* Encryption.m */; };
+		F89A1F5F13FAC8020071D321 /* NSObject_extra.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F5E13FAC8020071D321 /* NSObject_extra.m */; };
+		F89A1F6013FAC8020071D321 /* NSObject_extra.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F5E13FAC8020071D321 /* NSObject_extra.m */; };
+		F89A1F6313FAC8270071D321 /* CryptoKey.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F6213FAC8270071D321 /* CryptoKey.m */; };
+		F89A1F6413FAC8270071D321 /* CryptoKey.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F6213FAC8270071D321 /* CryptoKey.m */; };
+		F89A1F6713FAC83E0071D321 /* GunzipInputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F6613FAC83E0071D321 /* GunzipInputStream.m */; };
+		F89A1F6813FAC83E0071D321 /* GunzipInputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F6613FAC83E0071D321 /* GunzipInputStream.m */; };
+		F89A1F7713FAC8930071D321 /* NSData-GZip.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F7613FAC8930071D321 /* NSData-GZip.m */; };
+		F89A1F7813FAC8930071D321 /* NSData-GZip.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1F7613FAC8930071D321 /* NSData-GZip.m */; };
+		F89A1FD213FAD6BE0071D321 /* HSLog.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1FD113FAD6BE0071D321 /* HSLog.m */; };
+		F89A1FD313FAD6BE0071D321 /* HSLog.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A1FD113FAD6BE0071D321 /* HSLog.m */; };
+		F89A200B13FADAD70071D321 /* ArqSalt.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A200A13FADAD70071D321 /* ArqSalt.m */; };
+		F89A200C13FADAD70071D321 /* ArqSalt.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A200A13FADAD70071D321 /* ArqSalt.m */; };
+		F89A204613FAE29E0071D321 /* libz.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = F89A204513FAE29E0071D321 /* libz.dylib */; };
+		F89A205313FAE2DA0071D321 /* LocalS3Signer.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A205213FAE2DA0071D321 /* LocalS3Signer.m */; };
+		F89A205413FAE2DA0071D321 /* LocalS3Signer.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A205213FAE2DA0071D321 /* LocalS3Signer.m */; };
+		F89A205913FAE3010071D321 /* DataOutputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A205813FAE3010071D321 /* DataOutputStream.m */; };
+		F89A205A13FAE3010071D321 /* DataOutputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = F89A205813FAE3010071D321 /* DataOutputStream.m */; };
+		F89A20A513FAE5300071D321 /* libz.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = F89A20A413FAE5300071D321 /* libz.dylib */; };
+		F89A20CF13FAE7170071D321 /* NSError_extra.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F4D19C121D77E1002D09C1 /* NSError_extra.m */; };
 		F8D678001160F26A00CC270E /* Restorer.m in Sources */ = {isa = PBXBuildFile; fileRef = F8D677FF1160F26A00CC270E /* Restorer.m */; };
 		F8D6781D1160F4FD00CC270E /* SHA1Hash.m in Sources */ = {isa = PBXBuildFile; fileRef = F8D6781C1160F4FD00CC270E /* SHA1Hash.m */; };
-		F8D678331160F62E00CC270E /* ArqUserLibrary.m in Sources */ = {isa = PBXBuildFile; fileRef = F8D678321160F62E00CC270E /* ArqUserLibrary.m */; };
 		F8D6783C1160F70100CC270E /* PackIndexEntry.m in Sources */ = {isa = PBXBuildFile; fileRef = F8D6783B1160F70100CC270E /* PackIndexEntry.m */; };
 		F8D678451160F74A00CC270E /* DiskPack.m in Sources */ = {isa = PBXBuildFile; fileRef = F8D678421160F74A00CC270E /* DiskPack.m */; };
 		F8D678461160F74A00CC270E /* DiskPackIndex.m in Sources */ = {isa = PBXBuildFile; fileRef = F8D678441160F74A00CC270E /* DiskPackIndex.m */; };
@@ -203,12 +209,10 @@
 		F8F4D1E2121D7BC3002D09C1 /* AppKeychain.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F4D1E1121D7BC3002D09C1 /* AppKeychain.m */; };
 		F8F4D1E7121D7DA2002D09C1 /* FarkPath.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F4D1E6121D7DA2002D09C1 /* FarkPath.m */; };
 		F8F4D1FE121D8409002D09C1 /* PackIndexWriter.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F4D1FD121D8409002D09C1 /* PackIndexWriter.m */; };
-		F8F4D207121D8696002D09C1 /* ArqRepo_Verifier.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F4D206121D8696002D09C1 /* ArqRepo_Verifier.m */; };
 		F8F4D59A121D9FA7002D09C1 /* MonitoredInputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F4D19F121D78AD002D09C1 /* MonitoredInputStream.m */; };
 		F8F4D59B121D9FAD002D09C1 /* ArqFark.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F4D1C2121D79AC002D09C1 /* ArqFark.m */; };
 		F8F4D59C121D9FAE002D09C1 /* ArqPackSet.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F4D1AD121D7990002D09C1 /* ArqPackSet.m */; };
 		F8F4D59D121D9FAF002D09C1 /* ArqRepo.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F4D1AF121D7990002D09C1 /* ArqRepo.m */; };
-		F8F4D59E121D9FAF002D09C1 /* ArqRepo_Verifier.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F4D206121D8696002D09C1 /* ArqRepo_Verifier.m */; };
 		F8F4D5A1121D9FC2002D09C1 /* AppKeychain.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F4D1E1121D7BC3002D09C1 /* AppKeychain.m */; };
 		F8F4D5A2121D9FC9002D09C1 /* FarkPath.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F4D1E6121D7DA2002D09C1 /* FarkPath.m */; };
 		F8F4D5A3121D9FCF002D09C1 /* PackIndexWriter.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F4D1FD121D8409002D09C1 /* PackIndexWriter.m */; };
@@ -243,10 +247,6 @@
 		8DD76FA10486AA7600D96B5E /* arq_restore */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = arq_restore; sourceTree = BUILT_PRODUCTS_DIR; };
 		F805B54B1160D3E6007EC01E /* ArqRestoreCommand.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ArqRestoreCommand.h; sourceTree = "<group>"; };
 		F805B54C1160D3E6007EC01E /* ArqRestoreCommand.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ArqRestoreCommand.m; sourceTree = "<group>"; };
-		F805B71F1160D9C2007EC01E /* HSLog.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = HSLog.h; sourceTree = "<group>"; };
-		F805B7201160D9C2007EC01E /* HSLog.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = HSLog.m; sourceTree = "<group>"; };
-		F805B72E1160DBE9007EC01E /* ArqFolder.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ArqFolder.h; sourceTree = "<group>"; };
-		F805B72F1160DBE9007EC01E /* ArqFolder.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ArqFolder.m; sourceTree = "<group>"; };
 		F805B7411160DCFE007EC01E /* ArrayNode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ArrayNode.h; sourceTree = "<group>"; };
 		F805B7421160DCFE007EC01E /* ArrayNode.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ArrayNode.m; sourceTree = "<group>"; };
 		F805B7431160DCFE007EC01E /* BooleanNode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BooleanNode.h; sourceTree = "<group>"; };
@@ -265,14 +265,10 @@
 		F805B7501160DCFE007EC01E /* XMLPListReader.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = XMLPListReader.m; sourceTree = "<group>"; };
 		F805B7511160DCFE007EC01E /* XMLPListWriter.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = XMLPListWriter.h; sourceTree = "<group>"; };
 		F805B7521160DCFE007EC01E /* XMLPListWriter.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = XMLPListWriter.m; sourceTree = "<group>"; };
-		F805B7681160DD60007EC01E /* HTTPConnection_S3.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = HTTPConnection_S3.h; sourceTree = "<group>"; };
-		F805B7691160DD60007EC01E /* HTTPConnection_S3.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = HTTPConnection_S3.m; sourceTree = "<group>"; };
 		F805B76A1160DD60007EC01E /* NSError_S3.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NSError_S3.h; sourceTree = "<group>"; };
 		F805B76B1160DD60007EC01E /* NSError_S3.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = NSError_S3.m; sourceTree = "<group>"; };
 		F805B76C1160DD60007EC01E /* PathReceiver.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = PathReceiver.h; sourceTree = "<group>"; };
 		F805B76D1160DD60007EC01E /* PathReceiver.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = PathReceiver.m; sourceTree = "<group>"; };
-		F805B76E1160DD60007EC01E /* S3AuthorizationParameters.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = S3AuthorizationParameters.h; sourceTree = "<group>"; };
-		F805B76F1160DD60007EC01E /* S3AuthorizationParameters.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = S3AuthorizationParameters.m; sourceTree = "<group>"; };
 		F805B7701160DD60007EC01E /* S3AuthorizationProvider.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = S3AuthorizationProvider.h; sourceTree = "<group>"; };
 		F805B7711160DD60007EC01E /* S3AuthorizationProvider.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = S3AuthorizationProvider.m; sourceTree = "<group>"; };
 		F805B7721160DD60007EC01E /* S3Lister.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = S3Lister.h; sourceTree = "<group>"; };
@@ -286,19 +282,12 @@
 		F805B77C1160DD60007EC01E /* S3Request.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = S3Request.m; sourceTree = "<group>"; };
 		F805B77D1160DD60007EC01E /* S3Service.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = S3Service.h; sourceTree = "<group>"; };
 		F805B77E1160DD60007EC01E /* S3Service.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = S3Service.m; sourceTree = "<group>"; };
-		F805B77F1160DD60007EC01E /* S3Signature.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = S3Signature.h; sourceTree = "<group>"; };
-		F805B7801160DD60007EC01E /* S3Signature.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = S3Signature.m; sourceTree = "<group>"; };
 		F805B7A71160DEF2007EC01E /* RegexKitLite.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RegexKitLite.h; sourceTree = "<group>"; };
 		F805B7A81160DEF2007EC01E /* RegexKitLite.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = RegexKitLite.m; sourceTree = "<group>"; };
 		F805B7B81160E3AF007EC01E /* Blob.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Blob.h; sourceTree = "<group>"; };
 		F805B7B91160E3AF007EC01E /* Blob.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = Blob.m; sourceTree = "<group>"; };
 		F805B7CA1160E445007EC01E /* HTTP.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = HTTP.h; sourceTree = "<group>"; };
 		F805B7CB1160E445007EC01E /* HTTPConnection.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = HTTPConnection.h; sourceTree = "<group>"; };
-		F805B7CC1160E445007EC01E /* HTTPConnection.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = HTTPConnection.m; sourceTree = "<group>"; };
-		F805B7CD1160E445007EC01E /* HTTPRequest.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = HTTPRequest.h; sourceTree = "<group>"; };
-		F805B7CE1160E445007EC01E /* HTTPRequest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = HTTPRequest.m; sourceTree = "<group>"; };
-		F805B7CF1160E445007EC01E /* HTTPResponse.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = HTTPResponse.h; sourceTree = "<group>"; };
-		F805B7D01160E445007EC01E /* HTTPResponse.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = HTTPResponse.m; sourceTree = "<group>"; };
 		F805B7D61160E456007EC01E /* BlobACL.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BlobACL.h; sourceTree = "<group>"; };
 		F805B7D71160E456007EC01E /* BlobACL.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BlobACL.m; sourceTree = "<group>"; };
 		F805B7DE1160E48B007EC01E /* NSErrorCodes.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NSErrorCodes.h; sourceTree = "<group>"; };
@@ -330,12 +319,6 @@
 		F805B83C1160E900007EC01E /* StreamPair.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = StreamPair.h; sourceTree = "<group>"; };
 		F805B8401160E90F007EC01E /* Streams.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Streams.h; sourceTree = "<group>"; };
 		F805B8411160E90F007EC01E /* Streams.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = Streams.m; sourceTree = "<group>"; };
-		F805B8561160E9C9007EC01E /* CFStreamInputStream.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CFStreamInputStream.h; sourceTree = "<group>"; };
-		F805B8571160E9C9007EC01E /* CFStreamInputStream.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = CFStreamInputStream.m; sourceTree = "<group>"; };
-		F805B8581160E9C9007EC01E /* CFStreamOutputStream.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CFStreamOutputStream.h; sourceTree = "<group>"; };
-		F805B8591160E9C9007EC01E /* CFStreamOutputStream.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = CFStreamOutputStream.m; sourceTree = "<group>"; };
-		F805B85E1160E9F0007EC01E /* DNS_SDErrors.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = DNS_SDErrors.h; sourceTree = "<group>"; };
-		F805B85F1160E9F0007EC01E /* DNS_SDErrors.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = DNS_SDErrors.m; sourceTree = "<group>"; };
 		F805B8631160EA15007EC01E /* NSXMLNode_extra.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NSXMLNode_extra.h; sourceTree = "<group>"; };
 		F805B8641160EA15007EC01E /* NSXMLNode_extra.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = NSXMLNode_extra.m; sourceTree = "<group>"; };
 		F805B8661160EA36007EC01E /* InputStreamFactory.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = InputStreamFactory.h; sourceTree = "<group>"; };
@@ -357,25 +340,53 @@
 		F83C1A6111CA7BD20001958F /* ArqVerifyCommand.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ArqVerifyCommand.h; sourceTree = "<group>"; };
 		F83C1A6211CA7BD20001958F /* ArqVerifyCommand.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ArqVerifyCommand.m; sourceTree = "<group>"; };
 		F83C1AD711CA7C170001958F /* arq_verify */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = arq_verify; sourceTree = BUILT_PRODUCTS_DIR; };
-		F83C1D0811CA929D0001958F /* BucketVerifier.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BucketVerifier.h; sourceTree = "<group>"; };
+		F83C1D0811CA929D0001958F /* BucketVerifier.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = BucketVerifier.h; path = s3/BucketVerifier.h; sourceTree = "<group>"; };
-		F83C1D0911CA929D0001958F /* BucketVerifier.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BucketVerifier.m; sourceTree = "<group>"; };
+		F83C1D0911CA929D0001958F /* BucketVerifier.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = BucketVerifier.m; path = s3/BucketVerifier.m; sourceTree = "<group>"; };
 		F8987231121EB68900F07D76 /* BinaryPListReader.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BinaryPListReader.h; sourceTree = "<group>"; };
 		F8987232121EB68900F07D76 /* BinaryPListReader.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BinaryPListReader.m; sourceTree = "<group>"; };
 		F8987233121EB68900F07D76 /* BinaryPListWriter.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BinaryPListWriter.h; sourceTree = "<group>"; };
 		F8987234121EB68900F07D76 /* BinaryPListWriter.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BinaryPListWriter.m; sourceTree = "<group>"; };
-		F8987525121EB89100F07D76 /* CFStreamPair.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CFStreamPair.h; sourceTree = "<group>"; };
-		F8987526121EB89100F07D76 /* CFStreamPair.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = CFStreamPair.m; sourceTree = "<group>"; };
-		F898752F121EB94000F07D76 /* StreamPair.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = StreamPair.h; sourceTree = "<group>"; };
-		F8987530121EB94000F07D76 /* StreamPairFactory.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = StreamPairFactory.h; sourceTree = "<group>"; };
-		F8987531121EB94000F07D76 /* StreamPairFactory.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = StreamPairFactory.m; sourceTree = "<group>"; };
 		F898755F121EBD9600F07D76 /* BufferedInputStream.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BufferedInputStream.m; sourceTree = "<group>"; };
+		F89A1EEA13FAC4E30071D321 /* URLConnection.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = URLConnection.h; sourceTree = "<group>"; };
+		F89A1EEB13FAC4E30071D321 /* URLConnection.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = URLConnection.m; sourceTree = "<group>"; };
+		F89A1EF913FAC5970071D321 /* EncryptedInputStreamFactory.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = EncryptedInputStreamFactory.h; sourceTree = "<group>"; };
+		F89A1EFA13FAC5970071D321 /* EncryptedInputStreamFactory.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = EncryptedInputStreamFactory.m; sourceTree = "<group>"; };
+		F89A1F2913FAC6700071D321 /* UserLibrary.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = UserLibrary.h; sourceTree = "<group>"; };
+		F89A1F2A13FAC6700071D321 /* UserLibrary.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = UserLibrary.m; sourceTree = "<group>"; };
+		F89A1F3C13FAC6820071D321 /* UserLibrary_Arq.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = UserLibrary_Arq.h; sourceTree = "<group>"; };
+		F89A1F3D13FAC6820071D321 /* UserLibrary_Arq.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = UserLibrary_Arq.m; sourceTree = "<group>"; };
+		F89A1F4313FAC6C40071D321 /* BlobKey.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BlobKey.h; sourceTree = "<group>"; };
+		F89A1F4413FAC6C40071D321 /* BlobKey.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BlobKey.m; sourceTree = "<group>"; };
+		F89A1F4B13FAC73D0071D321 /* Computer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Computer.h; sourceTree = "<group>"; };
+		F89A1F4C13FAC73D0071D321 /* Computer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = Computer.m; sourceTree = "<group>"; };
+		F89A1F5113FAC78F0071D321 /* BufferedOutputStream.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BufferedOutputStream.h; sourceTree = "<group>"; };
+		F89A1F5213FAC78F0071D321 /* BufferedOutputStream.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BufferedOutputStream.m; sourceTree = "<group>"; };
+		F89A1F5913FAC7D50071D321 /* Encryption.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Encryption.h; sourceTree = "<group>"; };
+		F89A1F5A13FAC7D50071D321 /* Encryption.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = Encryption.m; sourceTree = "<group>"; };
+		F89A1F5D13FAC8020071D321 /* NSObject_extra.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NSObject_extra.h; sourceTree = "<group>"; };
+		F89A1F5E13FAC8020071D321 /* NSObject_extra.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = NSObject_extra.m; sourceTree = "<group>"; };
+		F89A1F6113FAC8270071D321 /* CryptoKey.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CryptoKey.h; sourceTree = "<group>"; };
+		F89A1F6213FAC8270071D321 /* CryptoKey.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = CryptoKey.m; sourceTree = "<group>"; };
+		F89A1F6513FAC83E0071D321 /* GunzipInputStream.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = GunzipInputStream.h; sourceTree = "<group>"; };
+		F89A1F6613FAC83E0071D321 /* GunzipInputStream.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GunzipInputStream.m; sourceTree = "<group>"; };
+		F89A1F7513FAC8930071D321 /* NSData-GZip.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "NSData-GZip.h"; sourceTree = "<group>"; };
+		F89A1F7613FAC8930071D321 /* NSData-GZip.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "NSData-GZip.m"; sourceTree = "<group>"; };
+		F89A1FD013FAD6BE0071D321 /* HSLog.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = HSLog.h; sourceTree = "<group>"; };
+		F89A1FD113FAD6BE0071D321 /* HSLog.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = HSLog.m; sourceTree = "<group>"; };
+		F89A200913FADAD70071D321 /* ArqSalt.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ArqSalt.h; sourceTree = "<group>"; };
+		F89A200A13FADAD70071D321 /* ArqSalt.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ArqSalt.m; sourceTree = "<group>"; };
+		F89A204513FAE29E0071D321 /* libz.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libz.dylib; path = usr/lib/libz.dylib; sourceTree = SDKROOT; };
+		F89A205113FAE2DA0071D321 /* LocalS3Signer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = LocalS3Signer.h; sourceTree = "<group>"; };
+		F89A205213FAE2DA0071D321 /* LocalS3Signer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = LocalS3Signer.m; sourceTree = "<group>"; };
+		F89A205713FAE3010071D321 /* DataOutputStream.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = DataOutputStream.h; sourceTree = "<group>"; };
+		F89A205813FAE3010071D321 /* DataOutputStream.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = DataOutputStream.m; sourceTree = "<group>"; };
+		F89A207F13FAE3810071D321 /* S3Signer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = S3Signer.h; sourceTree = "<group>"; };
+		F89A20A413FAE5300071D321 /* libz.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libz.dylib; path = usr/lib/libz.dylib; sourceTree = SDKROOT; };
 		F8D6763E1160F22800CC270E /* SetNSError.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SetNSError.h; sourceTree = "<group>"; };
 		F8D677FE1160F26A00CC270E /* Restorer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Restorer.h; sourceTree = "<group>"; };
 		F8D677FF1160F26A00CC270E /* Restorer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = Restorer.m; sourceTree = "<group>"; };
 		F8D6781B1160F4FD00CC270E /* SHA1Hash.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SHA1Hash.h; sourceTree = "<group>"; };
 		F8D6781C1160F4FD00CC270E /* SHA1Hash.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SHA1Hash.m; sourceTree = "<group>"; };
-		F8D678311160F62E00CC270E /* ArqUserLibrary.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ArqUserLibrary.h; sourceTree = "<group>"; };
-		F8D678321160F62E00CC270E /* ArqUserLibrary.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ArqUserLibrary.m; sourceTree = "<group>"; };
 		F8D6783A1160F70100CC270E /* PackIndexEntry.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = PackIndexEntry.h; sourceTree = "<group>"; };
 		F8D6783B1160F70100CC270E /* PackIndexEntry.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = PackIndexEntry.m; sourceTree = "<group>"; };
 		F8D678411160F74A00CC270E /* DiskPack.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = DiskPack.h; sourceTree = "<group>"; };
@@ -450,8 +461,6 @@
 		F8F4D1E6121D7DA2002D09C1 /* FarkPath.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FarkPath.m; sourceTree = "<group>"; };
 		F8F4D1FC121D8409002D09C1 /* PackIndexWriter.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = PackIndexWriter.h; sourceTree = "<group>"; };
 		F8F4D1FD121D8409002D09C1 /* PackIndexWriter.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = PackIndexWriter.m; sourceTree = "<group>"; };
-		F8F4D205121D8696002D09C1 /* ArqRepo_Verifier.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ArqRepo_Verifier.h; sourceTree = "<group>"; };
-		F8F4D206121D8696002D09C1 /* ArqRepo_Verifier.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ArqRepo_Verifier.m; sourceTree = "<group>"; };
 		F8F4D67C121DA542002D09C1 /* UserAndComputer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = UserAndComputer.h; sourceTree = "<group>"; };
 		F8F4D67D121DA542002D09C1 /* UserAndComputer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = UserAndComputer.m; sourceTree = "<group>"; };
 /* End PBXFileReference section */
@@ -469,6 +478,7 @@
 				F805B8A11160EBAA007EC01E /* CoreFoundation.framework in Frameworks */,
 				F805B8C21160EC41007EC01E /* Security.framework in Frameworks */,
 				F805B8CE1160ECD7007EC01E /* CoreServices.framework in Frameworks */,
+				F89A204613FAE29E0071D321 /* libz.dylib in Frameworks */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -484,6 +494,7 @@
 				F83C1AD011CA7C170001958F /* CoreFoundation.framework in Frameworks */,
 				F83C1AD111CA7C170001958F /* Security.framework in Frameworks */,
 				F83C1AD211CA7C170001958F /* CoreServices.framework in Frameworks */,
+				F89A20A513FAE5300071D321 /* libz.dylib in Frameworks */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -496,6 +507,8 @@
 				08FB7795FE84155DC02AAC07 /* Source */,
 				08FB779DFE84155DC02AAC07 /* External Frameworks and Libraries */,
 				1AB674ADFE9D54B511CA2CBB /* Products */,
+				F89A204513FAE29E0071D321 /* libz.dylib */,
+				F89A20A413FAE5300071D321 /* libz.dylib */,
 			);
 			name = arq_restore;
 			sourceTree = "<group>";
@@ -504,62 +517,33 @@
 			isa = PBXGroup;
 			children = (
 				F805B8671160EA7C007EC01E /* crypto */,
-				F805B7A61160DEF2007EC01E /* shared */,
-				F805B7401160DCFE007EC01E /* plist */,
-				F805B8081160E7A1007EC01E /* io */,
 				F805B7C91160E445007EC01E /* http */,
+				F805B8081160E7A1007EC01E /* io */,
+				F805B7401160DCFE007EC01E /* plist */,
+				F89A1EB613FAC3750071D321 /* repo */,
 				F805B7651160DD60007EC01E /* s3 */,
-				F8F4D1FC121D8409002D09C1 /* PackIndexWriter.h */,
+				F805B7A61160DEF2007EC01E /* shared */,
-				F8F4D1FD121D8409002D09C1 /* PackIndexWriter.m */,
-				F8F4D67C121DA542002D09C1 /* UserAndComputer.h */,
-				F8F4D67D121DA542002D09C1 /* UserAndComputer.m */,
-				32A70AAB03705E1F00C91783 /* arq_restore_Prefix.pch */,
-				08FB7796FE84155DC02AAC07 /* arq_restore.m */,
-				F8F4D1C1121D79AC002D09C1 /* ArqFark.h */,
-				F8F4D1C2121D79AC002D09C1 /* ArqFark.m */,
-				F8F4D1E5121D7DA2002D09C1 /* FarkPath.h */,
-				F8F4D1E6121D7DA2002D09C1 /* FarkPath.m */,
-				F8F4D1AC121D7990002D09C1 /* ArqPackSet.h */,
-				F8F4D1AD121D7990002D09C1 /* ArqPackSet.m */,
-				F8F4D1AE121D7990002D09C1 /* ArqRepo.h */,
-				F8F4D1AF121D7990002D09C1 /* ArqRepo.m */,
-				F8F4D205121D8696002D09C1 /* ArqRepo_Verifier.h */,
-				F8F4D206121D8696002D09C1 /* ArqRepo_Verifier.m */,
-				F8D678311160F62E00CC270E /* ArqUserLibrary.h */,
-				F8D678321160F62E00CC270E /* ArqUserLibrary.m */,
-				F805B54B1160D3E6007EC01E /* ArqRestoreCommand.h */,
-				F805B54C1160D3E6007EC01E /* ArqRestoreCommand.m */,
-				F83C1A5F11CA7A6B0001958F /* arq_verify.m */,
-				F83C1A6111CA7BD20001958F /* ArqVerifyCommand.h */,
-				F83C1A6211CA7BD20001958F /* ArqVerifyCommand.m */,
-				F805B72E1160DBE9007EC01E /* ArqFolder.h */,
-				F805B72F1160DBE9007EC01E /* ArqFolder.m */,
-				F8D6785B1160F7CE00CC270E /* Commit.h */,
-				F8D6785C1160F7CF00CC270E /* Commit.m */,
-				F8D678AD1160FAD900CC270E /* CommitFailedFile.h */,
-				F8D678AE1160FAD900CC270E /* CommitFailedFile.m */,
-				F8D678411160F74A00CC270E /* DiskPack.h */,
-				F8D678421160F74A00CC270E /* DiskPack.m */,
-				F8D678431160F74A00CC270E /* DiskPackIndex.h */,
-				F8D678441160F74A00CC270E /* DiskPackIndex.m */,
-				F805B71F1160D9C2007EC01E /* HSLog.h */,
-				F805B7201160D9C2007EC01E /* HSLog.m */,
-				F8D678B61160FB2100CC270E /* Node.h */,
-				F8D678B71160FB2100CC270E /* Node.m */,
-				F8D6783A1160F70100CC270E /* PackIndexEntry.h */,
-				F8D6783B1160F70100CC270E /* PackIndexEntry.m */,
-				F8D677FE1160F26A00CC270E /* Restorer.h */,
-				F8D677FF1160F26A00CC270E /* Restorer.m */,
-				F8D67F6E1161443600CC270E /* RestoreNode.h */,
-				F8D67F6F1161443600CC270E /* RestoreNode.m */,
-				F8D6785D1160F7CF00CC270E /* Tree.h */,
-				F8D6785E1160F7CF00CC270E /* Tree.m */,
-				F8D67CE91161363A00CC270E /* FileAttributes.h */,
-				F8D67CEA1161363A00CC270E /* FileAttributes.m */,
-				F8D67CF01161366100CC270E /* XAttrSet.h */,
-				F8D67CF11161366100CC270E /* XAttrSet.m */,
 				F8F4D1E0121D7BC3002D09C1 /* AppKeychain.h */,
 				F8F4D1E1121D7BC3002D09C1 /* AppKeychain.m */,
+				F805B54B1160D3E6007EC01E /* ArqRestoreCommand.h */,
+				F805B54C1160D3E6007EC01E /* ArqRestoreCommand.m */,
+				F83C1A6111CA7BD20001958F /* ArqVerifyCommand.h */,
+				F83C1A6211CA7BD20001958F /* ArqVerifyCommand.m */,
+				08FB7796FE84155DC02AAC07 /* arq_restore.m */,
+				F83C1A5F11CA7A6B0001958F /* arq_verify.m */,
+				32A70AAB03705E1F00C91783 /* arq_restore_Prefix.pch */,
+				F89A1F4313FAC6C40071D321 /* BlobKey.h */,
+				F89A1F4413FAC6C40071D321 /* BlobKey.m */,
+				F83C1D0811CA929D0001958F /* BucketVerifier.h */,
+				F83C1D0911CA929D0001958F /* BucketVerifier.m */,
+				F8D67F6E1161443600CC270E /* RestoreNode.h */,
+				F8D67F6F1161443600CC270E /* RestoreNode.m */,
+				F8D677FE1160F26A00CC270E /* Restorer.h */,
+				F8D677FF1160F26A00CC270E /* Restorer.m */,
+				F8F4D67C121DA542002D09C1 /* UserAndComputer.h */,
+				F8F4D67D121DA542002D09C1 /* UserAndComputer.m */,
+				F89A1F3C13FAC6820071D321 /* UserLibrary_Arq.h */,
+				F89A1F3D13FAC6820071D321 /* UserLibrary_Arq.m */,
 			);
 			name = Source;
 			sourceTree = "<group>";
@@ -623,16 +607,12 @@
 		F805B7651160DD60007EC01E /* s3 */ = {
 			isa = PBXGroup;
 			children = (
-				F83C1D0811CA929D0001958F /* BucketVerifier.h */,
+				F89A205113FAE2DA0071D321 /* LocalS3Signer.h */,
-				F83C1D0911CA929D0001958F /* BucketVerifier.m */,
+				F89A205213FAE2DA0071D321 /* LocalS3Signer.m */,
-				F805B7681160DD60007EC01E /* HTTPConnection_S3.h */,
-				F805B7691160DD60007EC01E /* HTTPConnection_S3.m */,
 				F805B76A1160DD60007EC01E /* NSError_S3.h */,
 				F805B76B1160DD60007EC01E /* NSError_S3.m */,
 				F805B76C1160DD60007EC01E /* PathReceiver.h */,
 				F805B76D1160DD60007EC01E /* PathReceiver.m */,
-				F805B76E1160DD60007EC01E /* S3AuthorizationParameters.h */,
-				F805B76F1160DD60007EC01E /* S3AuthorizationParameters.m */,
 				F805B7701160DD60007EC01E /* S3AuthorizationProvider.h */,
 				F805B7711160DD60007EC01E /* S3AuthorizationProvider.m */,
 				F805B7721160DD60007EC01E /* S3Lister.h */,
@@ -646,8 +626,7 @@
 				F805B77C1160DD60007EC01E /* S3Request.m */,
 				F805B77D1160DD60007EC01E /* S3Service.h */,
 				F805B77E1160DD60007EC01E /* S3Service.m */,
-				F805B77F1160DD60007EC01E /* S3Signature.h */,
+				F89A207F13FAE3810071D321 /* S3Signer.h */,
-				F805B7801160DD60007EC01E /* S3Signature.m */,
 			);
 			path = s3;
 			sourceTree = "<group>";
@@ -655,19 +634,25 @@
 		F805B7A61160DEF2007EC01E /* shared */ = {
 			isa = PBXGroup;
 			children = (
-				F8F4D19B121D77E1002D09C1 /* NSError_extra.h */,
+				F89A1FD013FAD6BE0071D321 /* HSLog.h */,
-				F8F4D19C121D77E1002D09C1 /* NSError_extra.m */,
+				F89A1FD113FAD6BE0071D321 /* HSLog.m */,
+				F89A1F4B13FAC73D0071D321 /* Computer.h */,
+				F89A1F4C13FAC73D0071D321 /* Computer.m */,
 				F8D6788A1160F8E500CC270E /* BinarySHA1.h */,
 				F8D6788B1160F8E500CC270E /* BinarySHA1.m */,
-				F805B7D61160E456007EC01E /* BlobACL.h */,
-				F805B7D71160E456007EC01E /* BlobACL.m */,
 				F805B7B81160E3AF007EC01E /* Blob.h */,
 				F805B7B91160E3AF007EC01E /* Blob.m */,
-				F805B85E1160E9F0007EC01E /* DNS_SDErrors.h */,
+				F805B7D61160E456007EC01E /* BlobACL.h */,
-				F805B85F1160E9F0007EC01E /* DNS_SDErrors.m */,
+				F805B7D71160E456007EC01E /* BlobACL.m */,
 				F8D67D031161384100CC270E /* FileACL.h */,
 				F8D67D041161384100CC270E /* FileACL.m */,
+				F89A1F7513FAC8930071D321 /* NSData-GZip.h */,
+				F89A1F7613FAC8930071D321 /* NSData-GZip.m */,
 				F805B7DE1160E48B007EC01E /* NSErrorCodes.h */,
+				F8F4D19B121D77E1002D09C1 /* NSError_extra.h */,
+				F8F4D19C121D77E1002D09C1 /* NSError_extra.m */,
+				F89A1F5D13FAC8020071D321 /* NSObject_extra.h */,
+				F89A1F5E13FAC8020071D321 /* NSObject_extra.m */,
 				F8D678871160F8CD00CC270E /* NSString_extra.h */,
 				F8D678881160F8CD00CC270E /* NSString_extra.m */,
 				F805B8631160EA15007EC01E /* NSXMLNode_extra.h */,
@@ -681,6 +666,8 @@
 				F805B7DF1160E48B007EC01E /* ServerBlob.h */,
 				F805B7E01160E48B007EC01E /* ServerBlob.m */,
 				F8D6763E1160F22800CC270E /* SetNSError.h */,
+				F89A1F2913FAC6700071D321 /* UserLibrary.h */,
+				F89A1F2A13FAC6700071D321 /* UserLibrary.m */,
 			);
 			path = shared;
 			sourceTree = "<group>";
@@ -688,18 +675,10 @@
 		F805B7C91160E445007EC01E /* http */ = {
 			isa = PBXGroup;
 			children = (
-				F898752F121EB94000F07D76 /* StreamPair.h */,
+				F89A1EEA13FAC4E30071D321 /* URLConnection.h */,
-				F8987530121EB94000F07D76 /* StreamPairFactory.h */,
+				F89A1EEB13FAC4E30071D321 /* URLConnection.m */,
-				F8987531121EB94000F07D76 /* StreamPairFactory.m */,
-				F8987525121EB89100F07D76 /* CFStreamPair.h */,
-				F8987526121EB89100F07D76 /* CFStreamPair.m */,
 				F805B7CA1160E445007EC01E /* HTTP.h */,
 				F805B7CB1160E445007EC01E /* HTTPConnection.h */,
-				F805B7CC1160E445007EC01E /* HTTPConnection.m */,
-				F805B7CD1160E445007EC01E /* HTTPRequest.h */,
-				F805B7CE1160E445007EC01E /* HTTPRequest.m */,
-				F805B7CF1160E445007EC01E /* HTTPResponse.h */,
-				F805B7D01160E445007EC01E /* HTTPResponse.m */,
 				F805B7F91160E73D007EC01E /* RFC2616DateFormatter.h */,
 				F805B7FA1160E73D007EC01E /* RFC2616DateFormatter.m */,
 			);
@@ -709,14 +688,16 @@
 		F805B8081160E7A1007EC01E /* io */ = {
 			isa = PBXGroup;
 			children = (
+				F89A205713FAE3010071D321 /* DataOutputStream.h */,
+				F89A205813FAE3010071D321 /* DataOutputStream.m */,
+				F89A1F6113FAC8270071D321 /* CryptoKey.h */,
+				F89A1F6213FAC8270071D321 /* CryptoKey.m */,
 				F8D678A31160FA5F00CC270E /* BooleanIO.h */,
 				F8D678A41160FA5F00CC270E /* BooleanIO.m */,
 				F805B8331160E882007EC01E /* BufferedInputStream.h */,
 				F898755F121EBD9600F07D76 /* BufferedInputStream.m */,
-				F805B8561160E9C9007EC01E /* CFStreamInputStream.h */,
+				F89A1F5113FAC78F0071D321 /* BufferedOutputStream.h */,
-				F805B8571160E9C9007EC01E /* CFStreamInputStream.m */,
+				F89A1F5213FAC78F0071D321 /* BufferedOutputStream.m */,
-				F805B8581160E9C9007EC01E /* CFStreamOutputStream.h */,
-				F805B8591160E9C9007EC01E /* CFStreamOutputStream.m */,
 				F805B8211160E857007EC01E /* ChunkedInputStream.h */,
 				F805B8221160E857007EC01E /* ChunkedInputStream.m */,
 				F8D678721160F85D00CC270E /* CryptInputStream.h */,
@@ -733,6 +714,8 @@
 				F8D6786E1160F84600CC270E /* DecryptedInputStream.m */,
 				F8D6787C1160F8A000CC270E /* DoubleIO.h */,
 				F8D6787D1160F8A000CC270E /* DoubleIO.m */,
+				F89A1EF913FAC5970071D321 /* EncryptedInputStreamFactory.h */,
+				F89A1EFA13FAC5970071D321 /* EncryptedInputStreamFactory.m */,
 				F8D678A61160FA6A00CC270E /* EncryptedInputStream.h */,
 				F8D678A71160FA6A00CC270E /* EncryptedInputStream.m */,
 				F805B8271160E861007EC01E /* FDInputStream.h */,
@@ -747,6 +730,8 @@
 				F8D6789C1160FA3900CC270E /* FileOutputStream.m */,
 				F805B8231160E857007EC01E /* FixedLengthInputStream.h */,
 				F805B8241160E857007EC01E /* FixedLengthInputStream.m */,
+				F89A1F6513FAC83E0071D321 /* GunzipInputStream.h */,
+				F89A1F6613FAC83E0071D321 /* GunzipInputStream.m */,
 				F805B8661160EA36007EC01E /* InputStreamFactory.h */,
 				F805B8191160E838007EC01E /* InputStreams.h */,
 				F805B81A1160E838007EC01E /* InputStreams.m */,
@@ -774,6 +759,8 @@
 		F805B8671160EA7C007EC01E /* crypto */ = {
 			isa = PBXGroup;
 			children = (
+				F89A1F5913FAC7D50071D321 /* Encryption.h */,
+				F89A1F5A13FAC7D50071D321 /* Encryption.m */,
 				F8D6781B1160F4FD00CC270E /* SHA1Hash.h */,
 				F8D6781C1160F4FD00CC270E /* SHA1Hash.m */,
 				F805B8681160EA83007EC01E /* NSData-Base64Extensions.h */,
@@ -786,6 +773,43 @@
 			path = crypto;
 			sourceTree = "<group>";
 		};
+		F89A1EB613FAC3750071D321 /* repo */ = {
+			isa = PBXGroup;
+			children = (
+				F89A200913FADAD70071D321 /* ArqSalt.h */,
+				F89A200A13FADAD70071D321 /* ArqSalt.m */,
+				F8F4D1C1121D79AC002D09C1 /* ArqFark.h */,
+				F8F4D1C2121D79AC002D09C1 /* ArqFark.m */,
+				F8F4D1AC121D7990002D09C1 /* ArqPackSet.h */,
+				F8F4D1AD121D7990002D09C1 /* ArqPackSet.m */,
+				F8F4D1AE121D7990002D09C1 /* ArqRepo.h */,
+				F8F4D1AF121D7990002D09C1 /* ArqRepo.m */,
+				F8D6785B1160F7CE00CC270E /* Commit.h */,
+				F8D6785C1160F7CF00CC270E /* Commit.m */,
+				F8D678AD1160FAD900CC270E /* CommitFailedFile.h */,
+				F8D678AE1160FAD900CC270E /* CommitFailedFile.m */,
+				F8D678411160F74A00CC270E /* DiskPack.h */,
+				F8D678421160F74A00CC270E /* DiskPack.m */,
+				F8D678431160F74A00CC270E /* DiskPackIndex.h */,
+				F8D678441160F74A00CC270E /* DiskPackIndex.m */,
+				F8F4D1E5121D7DA2002D09C1 /* FarkPath.h */,
+				F8F4D1E6121D7DA2002D09C1 /* FarkPath.m */,
+				F8D67CE91161363A00CC270E /* FileAttributes.h */,
+				F8D67CEA1161363A00CC270E /* FileAttributes.m */,
+				F8D678B61160FB2100CC270E /* Node.h */,
+				F8D678B71160FB2100CC270E /* Node.m */,
+				F8D6783A1160F70100CC270E /* PackIndexEntry.h */,
+				F8D6783B1160F70100CC270E /* PackIndexEntry.m */,
+				F8F4D1FC121D8409002D09C1 /* PackIndexWriter.h */,
+				F8F4D1FD121D8409002D09C1 /* PackIndexWriter.m */,
+				F8D6785D1160F7CF00CC270E /* Tree.h */,
+				F8D6785E1160F7CF00CC270E /* Tree.m */,
+				F8D67CF01161366100CC270E /* XAttrSet.h */,
+				F8D67CF11161366100CC270E /* XAttrSet.m */,
+			);
+			name = repo;
+			sourceTree = "<group>";
+		};
 /* End PBXGroup section */
 
 /* Begin PBXNativeTarget section */
@@ -832,7 +856,14 @@
 			isa = PBXProject;
 			buildConfigurationList = 1DEB927808733DD40010E9CD /* Build configuration list for PBXProject "arq_restore" */;
 			compatibilityVersion = "Xcode 3.1";
+			developmentRegion = English;
 			hasScannedForEncodings = 1;
+			knownRegions = (
+				English,
+				Japanese,
+				French,
+				German,
+			);
 			mainGroup = 08FB7794FE84155DC02AAC07 /* arq_restore */;
 			projectDirPath = "";
 			projectRoot = "";
@@ -849,8 +880,6 @@
 			buildActionMask = 2147483647;
 			files = (
 				F805B54D1160D3E6007EC01E /* ArqRestoreCommand.m in Sources */,
-				F805B7211160D9C2007EC01E /* HSLog.m in Sources */,
-				F805B7301160DBE9007EC01E /* ArqFolder.m in Sources */,
 				F805B7531160DCFE007EC01E /* ArrayNode.m in Sources */,
 				F805B7541160DCFE007EC01E /* BooleanNode.m in Sources */,
 				F805B7551160DCFE007EC01E /* DictNode.m in Sources */,
@@ -859,22 +888,16 @@
 				F805B7581160DCFE007EC01E /* StringNode.m in Sources */,
 				F805B7591160DCFE007EC01E /* XMLPListReader.m in Sources */,
 				F805B75A1160DCFE007EC01E /* XMLPListWriter.m in Sources */,
-				F805B7821160DD60007EC01E /* HTTPConnection_S3.m in Sources */,
 				F805B7831160DD60007EC01E /* NSError_S3.m in Sources */,
 				F805B7841160DD60007EC01E /* PathReceiver.m in Sources */,
-				F805B7851160DD60007EC01E /* S3AuthorizationParameters.m in Sources */,
 				F805B7861160DD60007EC01E /* S3AuthorizationProvider.m in Sources */,
 				F805B7871160DD60007EC01E /* S3Lister.m in Sources */,
 				F805B7881160DD60007EC01E /* S3ObjectMetadata.m in Sources */,
 				F805B7891160DD60007EC01E /* S3ObjectReceiver.m in Sources */,
 				F805B78B1160DD60007EC01E /* S3Request.m in Sources */,
 				F805B78C1160DD60007EC01E /* S3Service.m in Sources */,
-				F805B78D1160DD60007EC01E /* S3Signature.m in Sources */,
 				F805B7A91160DEF2007EC01E /* RegexKitLite.m in Sources */,
 				F805B7BA1160E3AF007EC01E /* Blob.m in Sources */,
-				F805B7D11160E445007EC01E /* HTTPConnection.m in Sources */,
-				F805B7D21160E445007EC01E /* HTTPRequest.m in Sources */,
-				F805B7D31160E445007EC01E /* HTTPResponse.m in Sources */,
 				F805B7D81160E456007EC01E /* BlobACL.m in Sources */,
 				F805B7E11160E48B007EC01E /* ServerBlob.m in Sources */,
 				F805B7FB1160E73D007EC01E /* RFC2616DateFormatter.m in Sources */,
@@ -888,15 +911,11 @@
 				F805B8321160E878007EC01E /* Writer.m in Sources */,
 				F805B83B1160E8DD007EC01E /* NSData-InputStream.m in Sources */,
 				F805B8421160E90F007EC01E /* Streams.m in Sources */,
-				F805B85A1160E9C9007EC01E /* CFStreamInputStream.m in Sources */,
-				F805B85B1160E9C9007EC01E /* CFStreamOutputStream.m in Sources */,
-				F805B8601160E9F0007EC01E /* DNS_SDErrors.m in Sources */,
 				F805B8651160EA15007EC01E /* NSXMLNode_extra.m in Sources */,
 				F805B86A1160EA83007EC01E /* NSData-Base64Extensions.m in Sources */,
 				F805B86F1160EAC1007EC01E /* DataInputStreamFactory.m in Sources */,
 				F8D678001160F26A00CC270E /* Restorer.m in Sources */,
 				F8D6781D1160F4FD00CC270E /* SHA1Hash.m in Sources */,
-				F8D678331160F62E00CC270E /* ArqUserLibrary.m in Sources */,
 				F8D6783C1160F70100CC270E /* PackIndexEntry.m in Sources */,
 				F8D678451160F74A00CC270E /* DiskPack.m in Sources */,
 				F8D678461160F74A00CC270E /* DiskPackIndex.m in Sources */,
@@ -927,7 +946,6 @@
 				F8D67D081161384100CC270E /* OSStatusDescription.m in Sources */,
 				F8D67F701161443600CC270E /* RestoreNode.m in Sources */,
 				F83C1AE311CA7C7C0001958F /* arq_restore.m in Sources */,
-				F83C1D0A11CA929D0001958F /* BucketVerifier.m in Sources */,
 				F8F4D19D121D77E1002D09C1 /* NSError_extra.m in Sources */,
 				F8F4D1A0121D78AD002D09C1 /* MonitoredInputStream.m in Sources */,
 				F8F4D1B0121D7990002D09C1 /* ArqPackSet.m in Sources */,
@@ -936,13 +954,26 @@
 				F8F4D1E2121D7BC3002D09C1 /* AppKeychain.m in Sources */,
 				F8F4D1E7121D7DA2002D09C1 /* FarkPath.m in Sources */,
 				F8F4D1FE121D8409002D09C1 /* PackIndexWriter.m in Sources */,
-				F8F4D207121D8696002D09C1 /* ArqRepo_Verifier.m in Sources */,
 				F8F4D67E121DA542002D09C1 /* UserAndComputer.m in Sources */,
 				F8987235121EB68900F07D76 /* BinaryPListReader.m in Sources */,
 				F8987236121EB68900F07D76 /* BinaryPListWriter.m in Sources */,
-				F8987527121EB89100F07D76 /* CFStreamPair.m in Sources */,
-				F8987532121EB94000F07D76 /* StreamPairFactory.m in Sources */,
 				F8987560121EBD9600F07D76 /* BufferedInputStream.m in Sources */,
+				F89A1EEC13FAC4E30071D321 /* URLConnection.m in Sources */,
+				F89A1EFB13FAC5970071D321 /* EncryptedInputStreamFactory.m in Sources */,
+				F89A1F2B13FAC6700071D321 /* UserLibrary.m in Sources */,
+				F89A1F3E13FAC6820071D321 /* UserLibrary_Arq.m in Sources */,
+				F89A1F4513FAC6C40071D321 /* BlobKey.m in Sources */,
+				F89A1F4D13FAC73D0071D321 /* Computer.m in Sources */,
+				F89A1F5313FAC78F0071D321 /* BufferedOutputStream.m in Sources */,
+				F89A1F5B13FAC7D50071D321 /* Encryption.m in Sources */,
+				F89A1F5F13FAC8020071D321 /* NSObject_extra.m in Sources */,
+				F89A1F6313FAC8270071D321 /* CryptoKey.m in Sources */,
+				F89A1F6713FAC83E0071D321 /* GunzipInputStream.m in Sources */,
+				F89A1F7713FAC8930071D321 /* NSData-GZip.m in Sources */,
+				F89A1FD213FAD6BE0071D321 /* HSLog.m in Sources */,
+				F89A200B13FADAD70071D321 /* ArqSalt.m in Sources */,
+				F89A205413FAE2DA0071D321 /* LocalS3Signer.m in Sources */,
+				F89A205A13FAE3010071D321 /* DataOutputStream.m in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -952,8 +983,6 @@
 			files = (
 				F83C1AC811CA7C170001958F /* arq_verify.m in Sources */,
 				F83C1A7411CA7C170001958F /* ArqRestoreCommand.m in Sources */,
-				F83C1A7511CA7C170001958F /* HSLog.m in Sources */,
-				F83C1A7611CA7C170001958F /* ArqFolder.m in Sources */,
 				F83C1A7711CA7C170001958F /* ArrayNode.m in Sources */,
 				F83C1A7811CA7C170001958F /* BooleanNode.m in Sources */,
 				F83C1A7911CA7C170001958F /* DictNode.m in Sources */,
@@ -962,22 +991,16 @@
 				F83C1A7C11CA7C170001958F /* StringNode.m in Sources */,
 				F83C1A7D11CA7C170001958F /* XMLPListReader.m in Sources */,
 				F83C1A7E11CA7C170001958F /* XMLPListWriter.m in Sources */,
-				F83C1A7F11CA7C170001958F /* HTTPConnection_S3.m in Sources */,
 				F83C1A8011CA7C170001958F /* NSError_S3.m in Sources */,
 				F83C1A8111CA7C170001958F /* PathReceiver.m in Sources */,
-				F83C1A8211CA7C170001958F /* S3AuthorizationParameters.m in Sources */,
 				F83C1A8311CA7C170001958F /* S3AuthorizationProvider.m in Sources */,
 				F83C1A8411CA7C170001958F /* S3Lister.m in Sources */,
 				F83C1A8511CA7C170001958F /* S3ObjectMetadata.m in Sources */,
 				F83C1A8611CA7C170001958F /* S3ObjectReceiver.m in Sources */,
 				F83C1A8711CA7C170001958F /* S3Request.m in Sources */,
 				F83C1A8811CA7C170001958F /* S3Service.m in Sources */,
-				F83C1A8911CA7C170001958F /* S3Signature.m in Sources */,
 				F83C1A8A11CA7C170001958F /* RegexKitLite.m in Sources */,
 				F83C1A8B11CA7C170001958F /* Blob.m in Sources */,
-				F83C1A8C11CA7C170001958F /* HTTPConnection.m in Sources */,
-				F83C1A8D11CA7C170001958F /* HTTPRequest.m in Sources */,
-				F83C1A8E11CA7C170001958F /* HTTPResponse.m in Sources */,
 				F83C1A8F11CA7C170001958F /* BlobACL.m in Sources */,
 				F83C1A9011CA7C170001958F /* ServerBlob.m in Sources */,
 				F83C1A9111CA7C170001958F /* RFC2616DateFormatter.m in Sources */,
@@ -991,15 +1014,11 @@
 				F83C1A9911CA7C170001958F /* Writer.m in Sources */,
 				F83C1A9A11CA7C170001958F /* NSData-InputStream.m in Sources */,
 				F83C1A9C11CA7C170001958F /* Streams.m in Sources */,
-				F83C1A9E11CA7C170001958F /* CFStreamInputStream.m in Sources */,
-				F83C1A9F11CA7C170001958F /* CFStreamOutputStream.m in Sources */,
-				F83C1AA011CA7C170001958F /* DNS_SDErrors.m in Sources */,
 				F83C1AA111CA7C170001958F /* NSXMLNode_extra.m in Sources */,
 				F83C1AA211CA7C170001958F /* NSData-Base64Extensions.m in Sources */,
 				F83C1AA311CA7C170001958F /* DataInputStreamFactory.m in Sources */,
 				F83C1AA411CA7C170001958F /* Restorer.m in Sources */,
 				F83C1AA711CA7C170001958F /* SHA1Hash.m in Sources */,
-				F83C1AAA11CA7C170001958F /* ArqUserLibrary.m in Sources */,
 				F83C1AAB11CA7C170001958F /* PackIndexEntry.m in Sources */,
 				F83C1AAC11CA7C170001958F /* DiskPack.m in Sources */,
 				F83C1AAD11CA7C170001958F /* DiskPackIndex.m in Sources */,
@@ -1035,16 +1054,30 @@
 				F8F4D59B121D9FAD002D09C1 /* ArqFark.m in Sources */,
 				F8F4D59C121D9FAE002D09C1 /* ArqPackSet.m in Sources */,
 				F8F4D59D121D9FAF002D09C1 /* ArqRepo.m in Sources */,
-				F8F4D59E121D9FAF002D09C1 /* ArqRepo_Verifier.m in Sources */,
 				F8F4D5A1121D9FC2002D09C1 /* AppKeychain.m in Sources */,
 				F8F4D5A2121D9FC9002D09C1 /* FarkPath.m in Sources */,
 				F8F4D5A3121D9FCF002D09C1 /* PackIndexWriter.m in Sources */,
 				F8987588121EBDF300F07D76 /* BinaryPListReader.m in Sources */,
 				F8987589121EBDF400F07D76 /* BinaryPListWriter.m in Sources */,
 				F898758A121EBDFA00F07D76 /* BufferedInputStream.m in Sources */,
-				F898758B121EBDFD00F07D76 /* CFStreamPair.m in Sources */,
 				F898758C121EBE0600F07D76 /* UserAndComputer.m in Sources */,
-				F898758D121EBE0800F07D76 /* StreamPairFactory.m in Sources */,
+				F89A1EED13FAC4E30071D321 /* URLConnection.m in Sources */,
+				F89A1EFC13FAC5970071D321 /* EncryptedInputStreamFactory.m in Sources */,
+				F89A1F2C13FAC6700071D321 /* UserLibrary.m in Sources */,
+				F89A1F3F13FAC6820071D321 /* UserLibrary_Arq.m in Sources */,
+				F89A1F4613FAC6C40071D321 /* BlobKey.m in Sources */,
+				F89A1F4E13FAC73D0071D321 /* Computer.m in Sources */,
+				F89A1F5413FAC78F0071D321 /* BufferedOutputStream.m in Sources */,
+				F89A1F5C13FAC7D50071D321 /* Encryption.m in Sources */,
+				F89A1F6013FAC8020071D321 /* NSObject_extra.m in Sources */,
+				F89A1F6413FAC8270071D321 /* CryptoKey.m in Sources */,
+				F89A1F6813FAC83E0071D321 /* GunzipInputStream.m in Sources */,
+				F89A1F7813FAC8930071D321 /* NSData-GZip.m in Sources */,
+				F89A1FD313FAD6BE0071D321 /* HSLog.m in Sources */,
+				F89A200C13FADAD70071D321 /* ArqSalt.m in Sources */,
+				F89A205313FAE2DA0071D321 /* LocalS3Signer.m in Sources */,
+				F89A205913FAE3010071D321 /* DataOutputStream.m in Sources */,
+				F89A20CF13FAE7170071D321 /* NSError_extra.m in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

arq_verify.m

@@ -35,7 +35,7 @@
 #import "ArqVerifyCommand.h"
 
 static void printUsage(const char *exeName) {
-	fprintf(stderr, "usage: %s [s3_bucket_name [computer_uuid [folder_uuid]]]\n", exeName);
+	fprintf(stderr, "usage: %s [-v]  all | s3_bucket_name [computer_uuid [folder_uuid]]]\n", exeName);
 }
 int main (int argc, const char * argv[]) {
     NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
@@ -64,27 +64,37 @@ int main (int argc, const char * argv[]) {
 	NSString *encryptionPassword = [[[NSString alloc] initWithUTF8String:cEncryptionPassword] autorelease];
     ArqVerifyCommand *cmd = [[[ArqVerifyCommand alloc] initWithAccessKey:accessKey secretKey:secretKey encryptionPassword:encryptionPassword] autorelease];
     NSError *error = nil;
+    BOOL verbose = NO;
+    int index = 1;
+    if (argc > 1 && !strcmp(argv[1], "-v")) {
+        verbose = YES;
+        index++;
+    }
+    [cmd setVerbose:verbose];
 	BOOL ret = NO;
-    if (argc == 1) {
+    if ((argc - index) == 0) {
-		if (![cmd verifyAll:&error]) {
+        printUsage(exeName);
-			NSLog(@"%@", [error localizedDescription]);
+        goto main_error;
-			goto main_error;
+	} else if ((argc - index) == 1) {
-		}
+		if (!strcmp(argv[index], "-?") || !strcmp(argv[index], "-h")) {
-	} else if (argc == 2) {
-		if (!strcmp(argv[1], "-?") || !strcmp(argv[1], "-h")) {
 			printUsage(exeName);
 			goto main_error;
-		} else if (![cmd verifyS3BucketName:[NSString stringWithUTF8String:argv[1]] error:&error]) {
+        } else if (!strcmp(argv[index], "all")) {
+            if (![cmd verifyAll:&error]) {
+                NSLog(@"%@", [error localizedDescription]);
+                goto main_error;
+            }
+		} else if (![cmd verifyS3BucketName:[NSString stringWithUTF8String:argv[index]] error:&error]) {
 			NSLog(@"%@", [error localizedDescription]);
 			goto main_error;
 		}
-	} else if (argc == 3) {
+	} else if ((argc - index) == 2) {
-		if (![cmd verifyS3BucketName:[NSString stringWithUTF8String:argv[1]] computerUUID:[NSString stringWithUTF8String:argv[2]] error:&error]) {
+		if (![cmd verifyS3BucketName:[NSString stringWithUTF8String:argv[index]] computerUUID:[NSString stringWithUTF8String:argv[index+1]] error:&error]) {
 			NSLog(@"%@", [error localizedDescription]);
 			goto main_error;
 		}
-	} else if (argc == 4) {
+	} else if ((argc - index) == 3) {
-		if (![cmd verifyS3BucketName:[NSString stringWithUTF8String:argv[1]] computerUUID:[NSString stringWithUTF8String:argv[2]] bucketUUID:[NSString stringWithUTF8String:argv[3]] error:&error]) {
+		if (![cmd verifyS3BucketName:[NSString stringWithUTF8String:argv[index]] computerUUID:[NSString stringWithUTF8String:argv[index+1]] bucketUUID:[NSString stringWithUTF8String:argv[index+2]] error:&error]) {
 			NSLog(@"%@", [error localizedDescription]);
 			goto main_error;
 		}

crypto/Encryption.h

@@ -0,0 +1,16 @@
+//
+//  Encryption.h
+//  Arq
+//
+//  Created by Stefan Reitshamer on 7/15/11.
+//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#import <Cocoa/Cocoa.h>
+
+
+@interface Encryption : NSObject {
+
+}
++ (NSString *)errorDomain;
+@end

crypto/Encryption.m

@@ -0,0 +1,16 @@
+//
+//  Encryption.m
+//  Arq
+//
+//  Created by Stefan Reitshamer on 7/15/11.
+//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#import "Encryption.h"
+
+
+@implementation Encryption
++ (NSString *)errorDomain {
+    return @"EncryptionErrorDomain";
+}
+@end

crypto/NSData-Base64Extensions.h

@@ -24,5 +24,4 @@
 
 - (NSString *) encodeBase64;
 - (NSString *) encodeBase64WithNewlines: (BOOL) encodeWithNewlines;
-
 @end

crypto/NSData-Base64Extensions.m

@@ -42,7 +42,9 @@
     
     // Encode all the data
     BIO_write(mem, [self bytes], [self length]);
-    BIO_flush(mem);
+    if (BIO_flush(mem) < 1) {
+        HSLogError(@"BIO_flush error");
+    }
     
     // Create a new string from the data in the memory buffer
     char * base64Pointer;
@@ -53,5 +55,4 @@
     BIO_free_all(mem);
     return base64String;
 }
-
 @end

crypto/NSData-Encrypt.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -31,13 +31,10 @@
  */ 
 
 #import <Cocoa/Cocoa.h>
-
+@class CryptoKey;
-#define ARQ_DEFAULT_CIPHER_NAME @"aes256"
 
 @interface NSData (Encrypt)
-+ (NSString *)encryptErrorDomain;
+- (NSData *)encryptWithCryptoKey:(CryptoKey *)theCryptoKey error:(NSError **)error;
-+ (NSString *)decryptErrorDomain;
+- (NSData *)decryptWithCryptoKey:(CryptoKey *)theCryptoKey error:(NSError **)error;
-- (NSData *)encryptWithCipher:(NSString *)cipherName key:(NSString *)key error:(NSError **)error;
-- (NSData *)decryptWithCipher:(NSString *)cipherName key:(NSString *)key error:(NSError **)error;
 
 @end

crypto/NSData-Encrypt.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -33,164 +33,31 @@
 #import <openssl/evp.h>
 
 #import "NSData-Encrypt.h"
-#import "OpenSSL.h"
+#import "DataInputStream.h"
-#import "SetNSError.h"
+#import "EncryptedInputStream.h"
-
+#import "DecryptedInputStream.h"
-@interface Crypter : NSObject {
-    NSData *data;
-    const EVP_CIPHER *cipher;
-    EVP_CIPHER_CTX cipherContext;
-    unsigned char evp_key[EVP_MAX_KEY_LENGTH];
-    unsigned char iv[EVP_MAX_IV_LENGTH];
-}
-- (id)initWithCipher:(NSString *)cipherName key:(NSString *)key data:(NSData *)theData error:(NSError **)error;
-- (NSData *)encrypt:(NSError **)error;
-- (NSData *)decrypt:(NSError **)error;
-@end
-
-@implementation Crypter 
-- (id)initWithCipher:(NSString *)cipherName key:(NSString *)key data:(NSData *)theData error:(NSError **)error {
-    if (self = [super init]) {
-        BOOL ret = NO;
-        do {
-            data = [theData retain];
-            if ([data length] > 0) {
-                NSData *keyData = [key dataUsingEncoding:NSUTF8StringEncoding];
-                if ([keyData length] > EVP_MAX_KEY_LENGTH) {
-                    SETNSERROR([NSData encryptErrorDomain], -1, @"encryption key must be less than or equal to %d bytes", EVP_MAX_KEY_LENGTH);
-                    break;
-                }
-                if (![OpenSSL initializeSSL:error]) {
-                    break;
-                }
-                cipher = EVP_get_cipherbyname([cipherName UTF8String]);
-                if (!cipher) {
-                    SETNSERROR([NSData encryptErrorDomain], -1, @"failed to load %@ cipher: %@", cipherName, [OpenSSL errorMessage]);
-                    break;
-                }
-                
-                evp_key[0] = 0;
-                EVP_BytesToKey(cipher, EVP_md5(), NULL, [keyData bytes], [keyData length], 1, evp_key, iv);
-                
-                EVP_CIPHER_CTX_init(&cipherContext);
-            }
-            ret = YES;
-        } while(0);
-        if (!ret) {
-            [self release];
-            self = nil;
-        }
-    }
-    return self;
-}
-- (void)dealloc {
-    if ([data length] > 0) {
-        EVP_CIPHER_CTX_cleanup(&cipherContext);
-    }
-    [data release];
-    [super dealloc];
-}
-- (NSData *)encrypt:(NSError **)error {
-    if ([data length] == 0) {
-        return [NSData data];
-    }
-    
-    if (!EVP_EncryptInit(&cipherContext, cipher, evp_key, iv)) {
-        SETNSERROR([NSData encryptErrorDomain], -1, @"EVP_EncryptInit: %@",  [OpenSSL errorMessage]);
-        return nil;
-    }
-    EVP_CIPHER_CTX_set_key_length(&cipherContext, EVP_MAX_KEY_LENGTH);
-    
-    // Need room for data + cipher block size - 1.
-    unsigned char *outbuf = (unsigned char *)calloc([data length] + EVP_CIPHER_CTX_block_size(&cipherContext) - 1, sizeof(unsigned char));
-    
-    int outlen;
-    if (!EVP_EncryptUpdate(&cipherContext, outbuf, &outlen, [data bytes], [data length])) {
-        SETNSERROR([NSData encryptErrorDomain], -1, @"EVP_EncryptUpdate: %@",  [OpenSSL errorMessage]);
-        return nil;
-    }
-    int templen;
-    if (!EVP_EncryptFinal(&cipherContext, outbuf + outlen, &templen)) {
-        SETNSERROR([NSData encryptErrorDomain], -1, @"EVP_EncryptFinal: %@",  [OpenSSL errorMessage]);
-        return nil;
-    }
-    outlen += templen;
-    NSData *ret = [[[NSData alloc] initWithBytes:outbuf length:outlen] autorelease];
-    free(outbuf);
-    return ret;
-}
-- (NSData *)decrypt:(NSError **)error {
-    if ([data length] == 0) {
-        return [NSData data];
-    }
-    
-    int inlen = [data length];
-    unsigned char *input = (unsigned char *)[data bytes];
-    
-    // Check for 8-byte salt in encrypted data and skip it.
-    if (inlen > 8+8 && strncmp((const char *)input, "Salted__", 8) == 0) {
-        input += 16;
-        inlen -= 16;
-    }
-    
-    if (!EVP_DecryptInit(&cipherContext, cipher, evp_key, iv)) {
-        SETNSERROR([NSData decryptErrorDomain], -1, @"EVP_DecryptInit: %@", [OpenSSL errorMessage]);
-        return nil;
-    }
-    EVP_CIPHER_CTX_set_key_length(&cipherContext, EVP_MAX_KEY_LENGTH);
-    
-    // The data buffer passed to EVP_DecryptUpdate() should have sufficient room for
-    // (input_length + cipher_block_size) bytes unless the cipher block size is 1 in which
-    // case input_length bytes is sufficient.
-    unsigned char *outbuf;
-    if(EVP_CIPHER_CTX_block_size(&cipherContext) > 1) {
-        outbuf = (unsigned char *)calloc(inlen + EVP_CIPHER_CTX_block_size(&cipherContext), sizeof(unsigned char));
-    } else {
-        outbuf = (unsigned char *)calloc(inlen, sizeof(unsigned char));
-    }
-    
-    int outlen;
-    if (!EVP_DecryptUpdate(&cipherContext, outbuf, &outlen, input, inlen)) {
-        SETNSERROR([NSData decryptErrorDomain], -1, @"EVP_DecryptUpdate: %@", [OpenSSL errorMessage]);
-        return nil;
-    }
-    int templen;
-    if (!EVP_DecryptFinal(&cipherContext, outbuf + outlen, &templen)) {
-        SETNSERROR([NSData decryptErrorDomain], -1, @"EVP_DecryptFinal: %@", [OpenSSL errorMessage]);
-        return nil;
-    }
-    outlen += templen;
-    NSData *ret = [[[NSData alloc] initWithBytes:outbuf length:outlen] autorelease];
-    free(outbuf);
-    return ret;
-}
-@end
 
 @implementation NSData (Encrypt)
-+ (NSString *)encryptErrorDomain {
+- (NSData *)encryptWithCryptoKey:(CryptoKey *)theCryptoKey error:(NSError **)error {
-    return @"NSDataEncryptErrorDomain";
+    DataInputStream *dis = [[DataInputStream alloc] initWithData:self];
-}
+    EncryptedInputStream *encrypted = [[EncryptedInputStream alloc] initWithInputStream:dis cryptoKey:theCryptoKey error:error];
-+ (NSString *)decryptErrorDomain {
+    [dis release];
-    return @"NSDataDecryptErrorDomain";
+    if (encrypted == nil) {
-}
+        return nil;
-- (NSData *)encryptWithCipher:(NSString *)cipherName key:(NSString *)key error:(NSError **)error {
-    NSData *ret = nil;
-    Crypter *crypter = [[Crypter alloc] initWithCipher:cipherName key:key data:self error:error];
-    if (crypter != nil) {
-        ret = [crypter encrypt:error];
-        [crypter release];
     }
+    NSData *ret = [encrypted slurp:error];
+    [encrypted release];
     return ret;
 }
-
+- (NSData *)decryptWithCryptoKey:(CryptoKey *)theCryptoKey error:(NSError **)error {
-- (NSData *)decryptWithCipher:(NSString *)cipherName key:(NSString *)key error:(NSError **)error {    
+    DataInputStream *dis = [[DataInputStream alloc] initWithData:self];
-    NSData *ret = nil;
+    DecryptedInputStream *decrypted = [[DecryptedInputStream alloc] initWithInputStream:dis cryptoKey:theCryptoKey error:error];
-    Crypter *crypter = [[Crypter alloc] initWithCipher:cipherName key:key data:self error:error];
+    [dis release];
-    if (crypter != nil) {
+    if (decrypted == nil) {
-        ret = [crypter decrypt:error];
+        return nil;
-        [crypter release];
     }
+    NSData *ret = [decrypted slurp:error];
+    [decrypted release];
     return ret;
 }
-
 @end

crypto/OpenSSL.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  

crypto/OpenSSL.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -66,7 +66,8 @@ static SSL_CTX *ctx;
         if ([msg length] > 0) {
             [msg appendString:@"; "];
         }
-        [msg appendFormat:@"%s", ERR_error_string(err, NULL)];
+        HSLogTrace(@"%s", ERR_error_string(err, NULL));
+        [msg appendFormat:@"%s", ERR_reason_error_string(err)];
     }
     if ([msg length] == 0) {
         [msg appendString:@"(no error)"];

crypto/SHA1Hash.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  

crypto/SHA1Hash.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -39,11 +39,12 @@
 #import "Blob.h"
 #import "BufferedInputStream.h"
 
-#define MY_BUF_SIZE (8192)
+#define MY_BUF_SIZE (4096)
 
 @interface SHA1Hash (internal)
 + (NSString *)hashStream:(id <InputStream>)is error:(NSError **)error;
 + (NSString *)hashStream:(id <InputStream>)is streamLength:(unsigned long long *)streamLength error:(NSError **)error;
++ (BOOL)updateSHA1:(SHA_CTX *)ctx fromStream:(id <InputStream>)is length:(unsigned long long)theLength error:(NSError **)error;
 @end
 
 static NSString *digest2String(unsigned char *digest) {
@@ -80,7 +81,9 @@ static NSString *digest2String(unsigned char *digest) {
 + (NSString *)hashFile:(NSString *)path error:(NSError **)error {
     struct stat st;
     if (lstat([path fileSystemRepresentation], &st) == -1) {
-        SETNSERROR(@"UnixErrorDomain", errno, @"lstat(%@): %s", path, strerror(errno));
+        int errnum = errno;
+        HSLogError(@"lstat(%@) error %d: %s", path, errnum, strerror(errnum));
+        SETNSERROR(@"UnixErrorDomain", errnum, @"%@: %s", path, strerror(errnum));
         return NO;
     }
     unsigned long long length = (unsigned long long)st.st_size;
@@ -103,7 +106,7 @@ static NSString *digest2String(unsigned char *digest) {
             break;
         }
         if (ret == 0) {
-            SETNSERROR([SHA1Hash errorDomain], -1, @"unexpected EOF after %qu of %qu bytes", received, length);
+            SETNSERROR([SHA1Hash errorDomain], ERROR_EOF, @"unexpected EOF in %@ after %qu of %qu bytes", is, received, length);
             break;
         }
         SHA1_Update(&ctx, buf, ret);
@@ -146,4 +149,26 @@ static NSString *digest2String(unsigned char *digest) {
 	SHA1_Final(md, &ctx);
     return digest2String(md);
 }
++ (BOOL)updateSHA1:(SHA_CTX *)ctx fromStream:(id <InputStream>)is length:(unsigned long long)theLength error:(NSError **)error {
+    unsigned char *imageBuf = (unsigned char *)malloc(MY_BUF_SIZE);
+    uint64_t recvd = 0;
+    NSInteger ret = 0;
+    while (recvd < theLength) {
+        uint64_t needed = theLength - recvd;
+        uint64_t toRead = needed > MY_BUF_SIZE ? MY_BUF_SIZE : needed;
+        ret = [is read:imageBuf bufferLength:toRead error:error];
+        if (ret < 0) {
+            break;
+        }
+        if (ret == 0) {
+            SETNSERROR([SHA1Hash errorDomain], -1, @"unexpected EOF reading image data from %@", is);
+            break;
+        }
+        SHA1_Update(
+                    ctx, imageBuf, ret);
+        recvd += (uint64_t)ret;
+    }
+    free(imageBuf);
+    return ret > 0;
+}
 @end

http/CFStreamPair.h

@@ -1,51 +0,0 @@
-/*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
-
-#import <Cocoa/Cocoa.h>
-#import "StreamPair.h"
-@class CFStreamInputStream;
-@class CFStreamOutputStream;
-
-@interface CFStreamPair : NSObject <StreamPair> {
-    NSString *description;
-    CFStreamInputStream *is;
-    CFStreamOutputStream *os;
-    NSTimeInterval createTime;
-    NSTimeInterval maxLifetime;
-    BOOL closeRequested;
-    
-}
-+ (NSString *)errorDomain;
-+ (NSError *)NSErrorWithNetworkError:(CFErrorRef)err;
-- (id)initWithHost:(NSString *)theHost port:(int)thePort useSSL:(BOOL)isUseSSL maxLifetime:(NSTimeInterval)theMaxLifetime;
-
-@end

http/CFStreamPair.m

@@ -1,221 +0,0 @@
-/*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
-
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netdb.h>
-#import <SystemConfiguration/SystemConfiguration.h>
-#import "CFStreamPair.h"
-#import "CFStreamInputStream.h"
-#import "CFStreamOutputStream.h"
-#import "DNS_SDErrors.h"
-
-@implementation CFStreamPair
-+ (NSString *)errorDomain {
-    return @"CFStreamPairErrorDomain";
-}
-+ (NSError *)NSErrorWithNetworkError:(CFErrorRef)err {
-    NSString *localizedDescription = @"Network error";
-    NSString *domain = (NSString *)CFErrorGetDomain(err);
-    CFIndex code = CFErrorGetCode(err);
-    CFDictionaryRef userInfo = CFErrorCopyUserInfo(err);
-    if ([domain isEqualToString:(NSString *)kCFErrorDomainCFNetwork]) {
-        if (code == kCFHostErrorHostNotFound) {
-            localizedDescription = @"host not found";
-        } else if (code == kCFHostErrorUnknown) {
-            int gaiCode = 0;
-            if (CFNumberGetValue((CFNumberRef)CFDictionaryGetValue(userInfo, kCFGetAddrInfoFailureKey), kCFNumberIntType, &gaiCode)) {
-                HSLogDebug(@"Host lookup error: %s", gai_strerror(gaiCode));
-                localizedDescription = @"Could not connect to the Internet";
-            }
-        } else if (code == kCFSOCKSErrorUnknownClientVersion) {
-            localizedDescription = @"Unknown SOCKS client version";
-        } else if (code == kCFSOCKSErrorUnsupportedServerVersion) {
-            localizedDescription = [NSString stringWithFormat:@"Unsupported SOCKS server version (server requested version %@)", (NSString *)CFDictionaryGetValue(userInfo, kCFSOCKSVersionKey)];
-        } else if (code == kCFSOCKS4ErrorRequestFailed) {
-            localizedDescription = @"SOCKS4 request rejected or failed";
-        } else if (code == kCFSOCKS4ErrorIdentdFailed) {
-            localizedDescription = @"SOCKS4 server cannot connect to identd on the client";
-        } else if (code == kCFSOCKS4ErrorIdConflict) {
-            localizedDescription = @"SOCKS4 client and identd report different user IDs";
-        } else if (code == kCFSOCKS4ErrorUnknownStatusCode) {
-            localizedDescription = [NSString stringWithFormat:@"SOCKS4 error %@", (NSString *)CFDictionaryGetValue(userInfo, kCFSOCKSStatusCodeKey)];
-        } else if (code == kCFSOCKS5ErrorBadState) {
-            localizedDescription = @"SOCKS5 bad state";
-        } else if (code == kCFSOCKS5ErrorBadResponseAddr) {
-            localizedDescription = @"SOCKS5 bad credentials";
-        } else if (code == kCFSOCKS5ErrorBadCredentials) {
-            localizedDescription = @"SOCKS5 unsupported negotiation method";
-        } else if (code == kCFSOCKS5ErrorUnsupportedNegotiationMethod) {
-            localizedDescription = @"SOCKS5 unsupported negotiation method";
-        } else if (code == kCFSOCKS5ErrorNoAcceptableMethod) {
-            localizedDescription = @"SOCKS5 no acceptable method";
-        } else if (code == kCFNetServiceErrorUnknown) {
-            localizedDescription = @"Unknown Net Services error";
-        } else if (code == kCFNetServiceErrorCollision) {
-            localizedDescription = @"Net Services: collision";
-        } else if (code == kCFNetServiceErrorNotFound) {
-            localizedDescription = @"Net Services: not found";
-        } else if (code == kCFNetServiceErrorInProgress) {
-            localizedDescription = @"Net Services: in progress";
-        } else if (code == kCFNetServiceErrorBadArgument) {
-            localizedDescription = @"Net Services: bad argument";
-        } else if (code == kCFNetServiceErrorCancel) {
-            localizedDescription = @"Net Services: cancelled";
-        } else if (code == kCFNetServiceErrorInvalid) {
-            localizedDescription = @"Net Services: invalid";
-        } else if (code == kCFNetServiceErrorTimeout) {
-            localizedDescription = @"Net Services timeout";
-        } else if (code == kCFNetServiceErrorDNSServiceFailure) {
-            localizedDescription = @"Net Services DNS failure";
-            int dns_sdCode = 0;
-            if (CFNumberGetValue((CFNumberRef)CFDictionaryGetValue(userInfo, kCFDNSServiceFailureKey), kCFNumberIntType, &dns_sdCode)) {
-                localizedDescription = [NSString stringWithFormat:@"Net Services DNS failure: %@", [DNS_SDErrors descriptionForDNS_SDError:dns_sdCode]];
-            }
-        } else if (code == kCFFTPErrorUnexpectedStatusCode) {
-            localizedDescription = [NSString stringWithFormat:@"FTP error %@", (NSString *)CFDictionaryGetValue(userInfo, kCFFTPStatusCodeKey)];
-        } else if (code == kCFErrorHTTPAuthenticationTypeUnsupported) {
-            localizedDescription = @"HTTP authentication type unsupported";
-        } else if (code == kCFErrorHTTPBadCredentials) {
-            localizedDescription = @"bad HTTP credentials";
-        } else if (code == kCFErrorHTTPConnectionLost) {
-            localizedDescription = @"HTTP connection lost";
-        } else if (code == kCFErrorHTTPParseFailure) {
-            localizedDescription = @"HTTP parse failure";
-        } else if (code == kCFErrorHTTPRedirectionLoopDetected) {
-            localizedDescription = @"HTTP redirection loop detected";
-        } else if (code == kCFErrorHTTPBadURL) {
-            localizedDescription = @"bad HTTP URL";
-        } else if (code == kCFErrorHTTPProxyConnectionFailure) {
-            localizedDescription = @"HTTP proxy connection failure";
-        } else if (code == kCFErrorHTTPBadProxyCredentials) {
-            localizedDescription = @"bad HTTP proxy credentials";
-        } else if (code == kCFErrorPACFileError) {
-            localizedDescription = @"HTTP PAC file error";
-        }
-    } else if ([domain isEqualToString:@"NSPOSIXErrorDomain"] && code == ENOTCONN) {
-        localizedDescription = @"Lost connection to the Internet";
-    } else {
-        localizedDescription = [(NSString *)CFErrorCopyDescription(err) autorelease];
-    }
-    CFRelease(userInfo);
-    return [NSError errorWithDomain:[CFStreamPair errorDomain] code:code userInfo:[NSDictionary dictionaryWithObjectsAndKeys:localizedDescription, NSLocalizedDescriptionKey, nil]];
-}
-- (id)initWithHost:(NSString *)theHost port:(int)thePort useSSL:(BOOL)isUseSSL maxLifetime:(NSTimeInterval)theMaxLifetime {
-    if (self = [super init]) {
-        description = [[NSString alloc] initWithFormat:@"<CFStreamPair host=%@ ssl=%@>", theHost, (isUseSSL ? @"YES" : @"NO")];
-        CFReadStreamRef readStream;
-        CFWriteStreamRef writeStream;
-        CFStreamCreatePairWithSocketToHost(kCFAllocatorDefault, (CFStringRef)theHost, thePort, &readStream, &writeStream);
-        if (isUseSSL) {
-            NSDictionary *sslProperties = [NSDictionary dictionaryWithObjectsAndKeys:
-                                           (NSString *)kCFStreamSocketSecurityLevelNegotiatedSSL, kCFStreamSSLLevel,
-                                           kCFBooleanTrue, kCFStreamSSLAllowsExpiredCertificates,
-                                           kCFBooleanTrue, kCFStreamSSLAllowsExpiredRoots,
-                                           kCFBooleanTrue, kCFStreamSSLAllowsAnyRoot,
-                                           kCFBooleanFalse, kCFStreamSSLValidatesCertificateChain,
-                                           kCFNull, kCFStreamSSLPeerName,
-                                           nil];
-            
-            CFReadStreamSetProperty(readStream, kCFStreamPropertySSLSettings, sslProperties);
-            CFWriteStreamSetProperty(writeStream, kCFStreamPropertySSLSettings, sslProperties);
-        }
-        NSDictionary *proxyDict = (NSDictionary *)SCDynamicStoreCopyProxies(NULL);
-        if ([proxyDict objectForKey:(NSString *)kCFStreamPropertyHTTPProxyHost] != nil) {
-            CFReadStreamSetProperty(readStream, kCFStreamPropertyHTTPProxy, proxyDict);
-            CFWriteStreamSetProperty(writeStream, kCFStreamPropertyHTTPProxy, proxyDict);
-        }
-        if ([proxyDict objectForKey:(NSString *)kCFStreamPropertySOCKSProxyHost] != nil && [proxyDict objectForKey:(NSString *)kCFStreamPropertySOCKSProxyPort] != nil) {
-            CFReadStreamSetProperty(readStream, kCFStreamPropertySOCKSProxy, proxyDict);
-            CFWriteStreamSetProperty(writeStream, kCFStreamPropertySOCKSProxy, proxyDict);
-        }
-        [proxyDict release];
-        
-        is = [[CFStreamInputStream alloc] initWithCFReadStream:readStream];
-        os = [[CFStreamOutputStream alloc] initWithCFWriteStream:writeStream];
-        CFRelease(readStream);
-        CFRelease(writeStream);
-        createTime = [NSDate timeIntervalSinceReferenceDate];
-        maxLifetime = theMaxLifetime;
-    }
-    return self;
-}
-- (void)dealloc {
-    [description release];
-    [is release];
-    [os release];
-    [super dealloc];
-}
-- (void)setCloseRequested {
-    closeRequested = YES;
-}
-- (BOOL)isUsable {
-    if (closeRequested) {
-        HSLogTrace(@"%@ close requested; not reusing", self);
-        return NO;
-    }
-    if (([NSDate timeIntervalSinceReferenceDate] - createTime) > maxLifetime) {
-        HSLogTrace(@"%@ > %f seconds old; not reusing", self, maxLifetime);
-        return NO;
-    }
-    return YES;
-}
-
-#pragma mark InputStream
-- (NSInteger)read:(unsigned char *)buf bufferLength:(NSUInteger)bufferLength error:(NSError **)error {
-    return [is read:buf bufferLength:bufferLength error:error];
-}
-- (NSData *)slurp:(NSError **)error {
-    return [is slurp:error];
-}
-
-#pragma mark OutputStream
-- (BOOL)write:(const unsigned char *)buf length:(NSUInteger)len error:(NSError **)error {
-    NSError *myError = nil;
-    BOOL ret = [os write:buf length:len error:&myError];
-    if (error != NULL) {
-        *error = myError;
-    }
-    if (!ret && [[myError domain] isEqualToString:@"UnixErrorDomain"] && [myError code] == EPIPE) {
-        HSLogError(@"broken pipe"); //FIXME: This may not work with CFStream stuff.
-    }
-    return ret;
-}
-- (unsigned long long)bytesWritten {
-    return [os bytesWritten];
-}
-
-#pragma mark NSObject
-- (NSString *)description {
-    return description;
-}
-@end

http/HTTP.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -33,9 +33,11 @@
 
 #define HTTP_1_1 @"1.1"
 #define HTTP_OK (200)
+#define HTTP_NO_CONTENT (204)
 #define HTTP_INTERNAL_SERVER_ERROR (500)
 #define HTTP_FORBIDDEN (403)
 #define HTTP_BAD_REQUEST (400)
+#define HTTP_METHOD_NOT_ALLOWED (405)
 #define HTTP_CONFLICT (409)
 #define HTTP_REQUESTED_RANGE_NOT_SATISFIABLE (416)
 #define HTTP_LENGTH_REQUIRED (411)

http/HTTPConnection.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -32,32 +32,23 @@
 
 #import <Cocoa/Cocoa.h>
 #import "InputStream.h"
-@protocol StreamPair;
-@class BufferedInputStream;
-@class HTTPRequest;
-@class HTTPResponse;
 
-@interface HTTPConnection : NSObject {
+@protocol HTTPConnection <InputStream>
-    id <StreamPair> streamPair;
-    BufferedInputStream *bufferedInputStream;
-    HTTPRequest *request;
-    HTTPResponse *response;
-}
-- (id)initWithHost:(NSString *)theHost useSSL:(BOOL)isUseSSL error:(NSError **)error;
-- (id)initWithHost:(NSString *)theHost port:(int)thePort useSSL:(BOOL)isUseSSL error:(NSError **)error;
-- (void)setRequestMethod:(NSString *)theRequestMethod pathInfo:(NSString *)thePathInfo queryString:(NSString *)theQueryString protocol:(NSString *)theProtocol;
 - (void)setRequestHeader:(NSString *)value forKey:(NSString *)key;
 - (void)setRequestHostHeader;
-- (void)setRequestKeepAliveHeader;
 - (void)setRequestContentDispositionHeader:(NSString *)downloadName;
 - (void)setRFC822DateRequestHeader;
+- (NSString *)requestMethod;
+- (NSString *)requestPathInfo;
+- (NSString *)requestQueryString;
+- (NSArray *)requestHeaderKeys;
+- (NSString *)requestHeaderForKey:(NSString *)theKey;
 - (BOOL)executeRequest:(NSError **)error;
-- (BOOL)executeRequestWithBody:(id <InputStream>)bodyStream error:(NSError **)error;
+- (BOOL)executeRequestWithBody:(NSData *)theBody error:(NSError **)error;
 - (int)responseCode;
 - (NSString *)responseHeaderForKey:(NSString *)key;
-- (NSString *)responseMimeType;
+- (NSString *)responseContentType;
 - (NSString *)responseDownloadName;
 - (id <InputStream>)newResponseBodyStream:(NSError **)error;
-- (NSData *)slurpResponseBody:(NSError **)error;
-- (void)setCloseRequested;
 @end
+

http/HTTPConnection.m

@@ -1,152 +0,0 @@
-/*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
-
-#import "HTTPConnection.h"
-#import "HTTPRequest.h"
-#import "HTTPResponse.h"
-#import "StreamPairFactory.h"
-#import "StreamPair.h"
-#import "Streams.h"
-#import "RegexKitLite.h"
-#import "FDOutputStream.h"
-#import "FDInputStream.h"
-#import "BufferedInputStream.h"
-
-static int HTTP_DEFAULT_PORT = 80;
-static int HTTPS_DEFAULT_PORT = 443;
-
-@implementation HTTPConnection
-- (id)initWithHost:(NSString *)theHost useSSL:(BOOL)isUseSSL error:(NSError **)error {
-    return [self initWithHost:theHost port:(isUseSSL ? HTTPS_DEFAULT_PORT : HTTP_DEFAULT_PORT) useSSL:isUseSSL error:error];
-}
-- (id)initWithHost:(NSString *)theHost port:(int)thePort useSSL:(BOOL)isUseSSL error:(NSError **)error {
-    if (self = [super init]) {
-        streamPair = [[StreamPairFactory theFactory] newStreamPairToHost:theHost port:thePort useSSL:isUseSSL error:error];
-        if (streamPair == nil) {
-            [self release];
-            return nil;
-        }
-        bufferedInputStream = [[BufferedInputStream alloc] initWithUnderlyingStream:streamPair];
-        request = [[HTTPRequest alloc] initWithHost:theHost];
-        response = [[HTTPResponse alloc] init];
-    }
-    return self;
-}
-- (void)dealloc {
-    [streamPair release];
-    [bufferedInputStream release];
-    [request release];
-    [response release];
-    [super dealloc];
-}
-- (void)setRequestMethod:(NSString *)theRequestMethod pathInfo:(NSString *)thePathInfo queryString:(NSString *)theQueryString protocol:(NSString *)theProtocol {
-    [request setMethod:theRequestMethod];
-    [request setPathInfo:thePathInfo];
-    [request setQueryString:theQueryString];
-    [request setProtocol:theProtocol];
-}
-- (void)setRequestHeader:(NSString *)value forKey:(NSString *)key {
-    [request setHeader:value forKey:key];
-}
-- (void)setRequestHostHeader {
-    [request setHostHeader];
-}
-- (void)setRequestKeepAliveHeader {
-    [request setKeepAliveHeader];
-}
-- (void)setRequestContentDispositionHeader:(NSString *)downloadName {
-    [request setContentDispositionHeader:downloadName];
-}
-- (void)setRFC822DateRequestHeader {
-    [request setRFC822DateHeader];
-}
-- (BOOL)executeRequest:(NSError **)error {
-    return [self executeRequestWithBody:nil error:error];
-}
-- (BOOL)executeRequestWithBody:(id <InputStream>)bodyStream error:(NSError **)error {
-    if (![request write:streamPair error:error]) {
-        return NO;
-    }
-    unsigned long long written = 0;
-    if (bodyStream != nil && ![Streams transferFrom:bodyStream to:streamPair bytesWritten:&written error:error]) {
-        return NO;
-    }
-    HSLogTrace(@"wrote %qu-byte request body", written);
-    if (![response readHead:bufferedInputStream requestMethod:[request method] error:error]) {
-        return NO;
-    }
-    if ([[response headerForKey:@"Connection"] isEqualToString:@"Close"]) {
-        HSLogDebug(@"Connection: Close header received; requesting close on %@", streamPair);
-        [streamPair setCloseRequested];
-    }
-    if (![[response protocol] isEqualToString:@"1.1"]) {
-        HSLogDebug(@"protocol %@ HTTP response; requesting close on %@", [response protocol], streamPair);
-        [streamPair setCloseRequested];
-    }
-    return YES;
-}
-- (int)responseCode {
-    return [response code];
-}
-- (NSString *)responseHeaderForKey:(NSString *)key {
-    return [response headerForKey:key];
-}
-- (NSString *)responseMimeType {
-    return [response headerForKey:@"Content-Type"];
-}
-- (NSString *)responseDownloadName {
-    NSString *downloadName = nil;
-    NSString *contentDisposition = [response headerForKey:@"Content-Disposition"];
-    if (contentDisposition != nil) {
-        NSRange filenameRange = [contentDisposition rangeOfRegex:@"attachment;filename=(.+)" capture:1];
-        if (filenameRange.location != NSNotFound) {
-            downloadName = [contentDisposition substringWithRange:filenameRange];
-        }
-    }
-    return downloadName;
-}
-- (id <InputStream>)newResponseBodyStream:(NSError **)error {
-    return [response newResponseInputStream:bufferedInputStream error:error];
-}
-- (NSData *)slurpResponseBody:(NSError **)error {
-    id <InputStream> is = [self newResponseBodyStream:error];
-    if (is == nil) {
-        return nil;
-    }
-    NSData *data = [is slurp:error];
-    [is release];
-    return data;
-}
-- (void)setCloseRequested {
-    [streamPair setCloseRequested];
-}
-@end

http/HTTPRequest.h

@@ -1,61 +0,0 @@
-/*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
-
-#import <Cocoa/Cocoa.h>
-#import "OutputStream.h"
-@class RFC2616DateFormatter;
-
-@interface HTTPRequest : NSObject {
-    NSString *host;
-    NSString *method;
-    NSString *pathInfo;
-    NSString *queryString;
-    NSString *protocol;
-    NSMutableDictionary *headers;
-    RFC2616DateFormatter *dateFormatter;
-}
-@property (copy, readonly) NSString *host;
-@property (copy) NSString *method;
-@property (copy) NSString *pathInfo;
-@property (copy) NSString *queryString;
-@property (copy) NSString *protocol;
-
-- (id)initWithHost:(NSString *)host;
-- (void)setHeader:(NSString *)value forKey:(NSString *)key;
-- (void)setHostHeader;
-- (void)setKeepAliveHeader;
-- (void)setRFC822DateHeader;
-- (void)setContentDispositionHeader:(NSString *)downloadName;
-- (NSString *)headerForKey:(NSString *)key;
-- (NSArray *)allHeaderKeys;
-- (BOOL)write:(id <OutputStream>)os error:(NSError **)error;
-@end

http/HTTPRequest.m

@@ -1,132 +0,0 @@
-/*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
-
-#import "HTTPRequest.h"
-#import "Writer.h"
-#import "RFC2616DateFormatter.h"
-
-#define DEFAULT_HTTP_PORT (80)
-
-@implementation HTTPRequest
-@synthesize host, method, pathInfo, queryString, protocol;
-
-- (id)initWithHost:(NSString *)theHost {
-    if (self = [super init]) {
-        host = [theHost retain];
-        headers = [[NSMutableDictionary alloc] init];
-        dateFormatter = [[RFC2616DateFormatter alloc] init];
-    }
-    return self;
-}
-
-- (void)dealloc {
-    [host release];
-    [method release];
-    [pathInfo release];
-    [queryString release];
-    [protocol release];
-    [headers release];
-    [dateFormatter release];
-    [super dealloc];
-}
-- (void)setHeader:(NSString *)value forKey:(NSString *)key {
-    [headers setValue:value forKey:key];
-}
-- (void)setHostHeader {
-    [headers setValue:host forKey:@"Host"];
-}
-- (void)setKeepAliveHeader {
-    [headers setValue:@"keep-alive" forKey:@"Connection"];
-}
-- (void)setRFC822DateHeader {
-    [headers setValue:[dateFormatter rfc2616StringFromDate:[NSDate date]] forKey:@"Date"];
-}
-- (void)setContentDispositionHeader:(NSString *)downloadName {
-    if (downloadName != nil) {
-        NSString *encodedFilename = [NSString stringWithFormat:@"\"%@\"", [downloadName stringByReplacingOccurrencesOfString:@"\"" withString:@"\\\\\""]];
-        encodedFilename = [encodedFilename stringByReplacingOccurrencesOfString:@"\n" withString:@"\\n"];
-        NSString *contentDisposition = [NSString stringWithFormat:@"attachment;filename=%@", encodedFilename];
-        [self setHeader:contentDisposition forKey:@"Content-Disposition"];
-    }
-}
-- (NSString *)headerForKey:(NSString *)key {
-    return [headers objectForKey:key];
-}
-- (NSArray *)allHeaderKeys {
-    return [headers allKeys];
-}
-- (BOOL)write:(id <OutputStream>)os error:(NSError **)error {
-    HSLogTrace(@"writing %@", self);
-    Writer *writer = [[Writer alloc] initWithOutputStream:os];
-    BOOL ret = NO;
-    do {
-        HSLogTrace(@"writing %@ %@%@ HTTP/%@\\r\\n", method, pathInfo, (queryString != nil ? queryString : @""), protocol);
-        if (![writer write:method error:error]
-            || ![writer write:@" " error:error]
-            || ![writer write:pathInfo error:error]) {
-            break;
-        }
-        if (queryString != nil) {
-            HSLogTrace(@"writing %@", queryString);
-            if (![writer write:queryString error:error]) {
-                break;
-            }
-        }
-        if (![writer write:@" HTTP/" error:error] 
-            || ![writer write:protocol error:error]
-            || ![writer write:@"\r\n" error:error]) {
-            break;
-        }
-        for (NSString *key in [headers allKeys]) {
-            HSLogTrace(@"writing %@: %@\\r\\n", key, [headers objectForKey:key]);
-            if (![writer write:key error:error]
-                || ![writer write:@": " error:error]
-                || ![writer write:[headers objectForKey:key] error:error]
-                || ![writer write:@"\r\n" error:error]) {
-                break;
-            }
-        }
-        HSLogTrace(@"writing \\r\\n");
-        if (![writer write:@"\r\n" error:error]) {
-            break;
-        }
-        ret = YES;
-    } while(0);
-    [writer release];
-    return ret;
-}
-
-#pragma mark NSObject protocol
-- (NSString *)description {
-    return [NSString stringWithFormat:@"<HTTPRequest: %@ %@%@ HTTP/%@>", method, pathInfo, (queryString != nil ? queryString : @""), protocol];
-}
-@end

http/HTTPResponse.h

@@ -1,51 +0,0 @@
-/*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
-
-#import <Cocoa/Cocoa.h>
-@protocol InputStream;
-@class FDInputStream;
-@class BufferedInputStream;
-
-@interface HTTPResponse : NSObject {
-    int code;
-    NSString *protocol;
-    NSMutableDictionary *headers;
-    NSString *requestMethod;
-}
-- (id)init;
-- (BOOL)readHead:(BufferedInputStream *)is requestMethod:(NSString *)requestMethod error:(NSError **)error;
-- (int)code;
-- (NSString *)protocol;
-- (NSString *)headerForKey:(NSString *)key;
-- (unsigned long long)contentLength;
-- (id <InputStream>)newResponseInputStream:(BufferedInputStream *)underlyingStream error:(NSError **)error;
-@end

http/HTTPResponse.m

@@ -1,158 +0,0 @@
-/*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
-
-#import "HTTPResponse.h"
-#import "RegexKitLite.h"
-#import "SetNSError.h"
-#import "BufferedInputStream.h"
-#import "DataInputStream.h"
-#import "ChunkedInputStream.h"
-#import "FixedLengthInputStream.h"
-#import "InputStream.h"
-#import "InputStreams.h"
-#import "NSData-InputStream.h"
-#import "FDInputStream.h"
-
-#define MAX_HTTP_STATUS_LINE_LENGTH (8192)
-#define MAX_HTTP_HEADER_LINE_LENGTH (8192)
-
-@implementation HTTPResponse
-- (id)init {
-    if (self = [super init]) {
-        headers = [[NSMutableDictionary alloc] init];
-    }
-    return self;
-}
-- (void)dealloc {
-    [requestMethod release];
-    [protocol release];
-    [headers release];
-    [super dealloc];
-}
-- (int)code {
-    return code;
-}
-- (NSString *)protocol {
-    return protocol;
-}
-- (NSString *)headerForKey:(NSString *)key {
-    return [headers objectForKey:key];
-}
-- (unsigned long long)contentLength {
-    long long contentLength = 0;
-    NSString *str = [self headerForKey:@"Content-Length"];
-    if (str != nil) {
-        NSScanner *scanner = [NSScanner scannerWithString:str];
-        if (![scanner scanLongLong:&contentLength]) {
-            HSLogWarn(@"unable to scan Content-Length %@", str);
-        }
-    }
-    return (unsigned long long)contentLength;
-}
-- (id <InputStream>)newResponseInputStream:(BufferedInputStream *)underlyingStream error:(NSError **)error {
-    id <InputStream>ret = nil;
-    if ([requestMethod isEqualToString:@"HEAD"] || code == 204) {
-        ret = [[NSData data] newInputStream];
-    } else {
-        NSString *transferEncoding = [self headerForKey:@"Transfer-Encoding"];
-        NSString *contentLength = [self headerForKey:@"Content-Length"];
-        if (transferEncoding != nil) {
-            if ([[transferEncoding lowercaseString] isEqualToString:@"chunked"]) {
-                ret = [[ChunkedInputStream alloc] initWithUnderlyingStream:underlyingStream];
-            } else {
-                SETNSERROR(@"StreamErrorDomain", -1, @"unknown Transfer-Encoding: %@", transferEncoding);
-            }
-        } else if (contentLength != nil) {
-            int length = [contentLength intValue];
-            ret = [[FixedLengthInputStream alloc] initWithUnderlyingStream:underlyingStream length:(NSUInteger)length];
-        } else {
-            /* 
-             * FIXME: handle multipart/byteranges media type.
-             * See rfc2616 section 4.4 ("message length").
-             */
-            HSLogWarn(@"response body with no content-length");
-            ret = [underlyingStream retain];
-        }
-    }
-    return ret;
-}
-- (BOOL)readHead:(BufferedInputStream *)inputStream requestMethod:(NSString *)theRequestMethod error:(NSError **)error {
-    [headers removeAllObjects];
-    [requestMethod release];
-    requestMethod = [theRequestMethod copy];
-    HSLogTrace(@"reading response start line");
-    NSString *line = [InputStreams readLineWithCRLF:inputStream maxLength:MAX_HTTP_STATUS_LINE_LENGTH error:error];
-    if (line == nil) {
-        return NO;
-    }
-    HSLogTrace(@"response start line: %@", line);
-    NSString *pattern = @"^HTTP/(1.\\d)\\s+(\\d+)\\s+(.+)\r\n$";
-    NSRange protoRange = [line rangeOfRegex:pattern capture:1];
-    NSRange codeRange = [line rangeOfRegex:pattern capture:2];
-    if (protoRange.location == NSNotFound || codeRange.location == NSNotFound) {
-        SETNSERROR(@"HTTPResponseErrorDomain", -1, @"unexpected response status line: %@", line);
-        return NO;
-    }
-    protocol = [[line substringWithRange:protoRange] retain];
-    code = [[line substringWithRange:codeRange] intValue];
-    
-    NSString *headerPattern = @"^([^:]+):\\s+(.+)\r\n$";
-    for(;;) {
-        line = [InputStreams readLineWithCRLF:inputStream maxLength:MAX_HTTP_HEADER_LINE_LENGTH error:error];
-        if (line == nil) {
-            return NO;
-        }
-        HSLogTrace(@"response header: %@", line);
-        if ([line isEqualToString:@"\r\n"]) {
-            break;
-        }
-        NSRange nameRange = [line rangeOfRegex:headerPattern capture:1];
-        NSRange valueRange = [line rangeOfRegex:headerPattern capture:2];
-        if (nameRange.location == NSNotFound || valueRange.location == NSNotFound) {
-            SETNSERROR(@"HTTPResponseErrorDomain", -1, @"invalid response header: %@", line);
-            return NO;
-        }
-        NSString *name = [line substringWithRange:nameRange];
-        NSString *value = [line substringWithRange:valueRange];
-        if ([headers objectForKey:name] != nil) {
-            HSLogWarn(@"dumping response header %@ = %@", name, [headers objectForKey:name]);
-        }
-        [headers setObject:value forKey:name];
-    }
-    return YES;
-}
-
-#pragma mark NSObject protocol
-- (NSString *)description {
-    return [NSString stringWithFormat:@"<HTTPResponse %p: code=%d headers=%@>", self, code, headers];
-}
-@end

http/RFC2616DateFormatter.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  

http/RFC2616DateFormatter.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  

http/StreamPairFactory.m

@@ -1,177 +0,0 @@
-/*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
-
-#import "StreamPairFactory.h"
-#import "CFStreamPair.h"
-
-static StreamPairFactory *theFactory = nil;
-
-#define DEFAULT_MAX_STREAM_PAIR_LIFETIME_SECONDS (60)
-#define CLEANUP_THREAD_SLEEP_SECONDS (5)
-
-@interface StreamPairMap : NSObject {
-    NSMutableDictionary *streamPairs;
-}
-- (id <StreamPair>)newStreamPairToHost:(NSString *)host port:(int)thePort useSSL:(BOOL)useSSL maxLifeTimeSeconds:(NSTimeInterval)theMaxLifetime error:(NSError **)error;
-- (void)dropUnusableStreamPairs;
-@end
-
-@implementation StreamPairMap
-- (id)init {
-    if (self = [super init]) {
-        streamPairs = [[NSMutableDictionary alloc] init];
-    }
-    return self;
-}
-- (void)dealloc {
-    [streamPairs release];
-    [super dealloc];
-}
-- (id <StreamPair>)newStreamPairToHost:(NSString *)host port:(int)thePort useSSL:(BOOL)useSSL maxLifeTimeSeconds:(NSTimeInterval)theMaxLifetime error:(NSError **)error {
-    NSString *key = [NSString stringWithFormat:@"%@:%d:%@", [host lowercaseString], thePort, (useSSL ? @"SSL" : @"noSSL")];
-    id <StreamPair> streamPair = [streamPairs objectForKey:key];
-    if (streamPair != nil) {
-        if (![streamPair isUsable]) {
-            [streamPairs removeObjectForKey:key];
-            streamPair = nil;
-        } else {
-            [streamPair retain];
-        }
-    }
-    if (streamPair == nil) {
-        streamPair = [[CFStreamPair alloc] initWithHost:host port:thePort useSSL:useSSL maxLifetime:theMaxLifetime];
-        [streamPairs setObject:streamPair forKey:key];
-    }
-    return streamPair;
-}
-- (void)dropUnusableStreamPairs {
-    NSMutableArray *keysToDrop = [NSMutableArray array];
-    for (NSString *key in streamPairs) {
-        id <StreamPair> streamPair = [streamPairs objectForKey:key];
-        if (![streamPair isUsable]) {
-            [keysToDrop addObject:key];
-        }
-    }
-    [streamPairs removeObjectsForKeys:keysToDrop];
-}
-@end
-
-
-
-@implementation StreamPairFactory
-+ (StreamPairFactory *)theFactory {
-    if (theFactory == nil) {
-        theFactory = [[super allocWithZone:NULL] init];
-    }
-    return theFactory;
-}
-
-/* Singleton recipe: */
-+ (id)allocWithZone:(NSZone *)zone {
-    return [[StreamPairFactory theFactory] retain];
-}
-- (id)copyWithZone:(NSZone *)zone {
-    return self;
-}
-- (id)retain {
-    return self;
-}
-- (NSUInteger)retainCount {
-    return NSUIntegerMax;  //denotes an object that cannot be released
-}
-- (void)release {
-    //do nothing
-}
-- (id)autorelease {
-    return self;
-}
-
-- (id)init {
-    if (self = [super init]) {
-        lock = [[NSLock alloc] init];
-        [lock setName:@"SocketFactory lock"];
-        threadMap = [[NSMutableDictionary alloc] init];
-        maxStreamPairLifetime = DEFAULT_MAX_STREAM_PAIR_LIFETIME_SECONDS;
-        [NSThread detachNewThreadSelector:@selector(dropUnusableSockets) toTarget:self withObject:nil];
-    }
-    return self;
-}
-- (void)dealloc {
-    [lock release];
-    [threadMap release];
-    [super dealloc];
-}
-- (void)setMaxStreamPairLifetime:(NSTimeInterval)theMaxLifetime {
-    maxStreamPairLifetime = theMaxLifetime;
-}
-- (id <StreamPair>)newStreamPairToHost:(NSString *)theHost port:(int)thePort useSSL:(BOOL)isUseSSL error:(NSError **)error {
-    void *pthreadPtr = pthread_self();
-#ifdef __LP64__
-    NSNumber *threadID = [NSNumber numberWithUnsignedLongLong:(uint64_t)pthreadPtr];
-#else
-    NSNumber *threadID = [NSNumber numberWithUnsignedLong:(uint32_t)pthreadPtr];
-#endif
-    [lock lock];
-    StreamPairMap *map = [threadMap objectForKey:threadID];
-    if (map == nil) {
-        map = [[StreamPairMap alloc] init];
-        [threadMap setObject:map forKey:threadID];
-        [map release];
-    }
-    id <StreamPair> streamPair = [map newStreamPairToHost:theHost port:thePort useSSL:isUseSSL maxLifeTimeSeconds:maxStreamPairLifetime error:error];
-    [lock unlock];
-    return streamPair;
-}
-- (void)clear {
-    [lock lock];
-    [threadMap removeAllObjects];
-    [lock unlock];
-}
-
-#pragma mark cleanup thread
-- (void)dropUnusableSockets {
-    for (;;) {
-        NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
-        @try {
-            [NSThread sleepForTimeInterval:CLEANUP_THREAD_SLEEP_SECONDS];
-            [lock lock];
-            for (StreamPairMap *map in [threadMap allValues]) {
-                [map dropUnusableStreamPairs];
-            }
-            [lock unlock];
-        } @catch(NSException *e) {
-            HSLogError(@"unexpected exception in StreamPairFactory cleanup thread: %@", [e description]);
-        }
-        [pool drain];
-    }
-}
-@end

http/URLConnection.h

@@ -0,0 +1,42 @@
+//
+//  URLConnection.h
+//  Arq
+//
+//  Created by Stefan Reitshamer on 5/3/11.
+//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#import <Cocoa/Cocoa.h>
+@class RFC2616DateFormatter;
+#import "HTTPConnection.h"
+
+#define THROTTLE_NONE 0
+#define THROTTLE_AUTOMATIC 1
+#define THROTTLE_FIXED 2
+
+@interface URLConnection : NSObject <HTTPConnection> {
+    NSString *method;
+    id delegate;
+    NSMutableURLRequest *mutableURLRequest;
+    NSURLConnection *urlConnection;
+    NSHTTPURLResponse *httpURLResponse;
+    RFC2616DateFormatter *dateFormatter;
+    BOOL complete;
+    unsigned long long totalSent;
+    NSData *receivedData;
+    NSUInteger offset;
+    NSUInteger totalReceived;
+    NSError *_error;
+    BOOL responseHasContentLength;
+    NSUInteger contentLength;
+    NSTimeInterval lastSentTime;
+}
++ (NSString *)errorDomain;
+
+- (id)initWithURL:(NSURL *)theURL method:(NSString *)theMethod delegate:(id)theDelegate;
+@end
+
+@interface NSObject (URLConnectionDelegate)
+- (void)urlConnection:(URLConnection *)theURLConnection sentBytes:(unsigned long long)sent throttleType:(int *)theThrottleType throttleKBPS:(int *)theThrottleKBPS pauseRequested:(BOOL *)isPauseRequested abortRequested:(BOOL *)isAbortRequested;
+- (void)urlConnection:(URLConnection *)theURLConnection subtractSentBytes:(unsigned long long)sent;
+@end

http/URLConnection.m

@@ -0,0 +1,276 @@
+//
+//  URLConnection.m
+//  Arq
+//
+//  Created by Stefan Reitshamer on 5/3/11.
+//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#import "URLConnection.h"
+#import "RFC2616DateFormatter.h"
+#import "InputStream.h"
+#import "RegexKitLite.h"
+#import "NSData-InputStream.h"
+#import "ChunkedInputStream.h"
+#import "SetNSError.h"
+#import "DataInputStream.h"
+#import "BufferedInputStream.h"
+#import "InputStreams.h"
+#import "NSErrorCodes.h"
+#import "NSError_extra.h"
+#import "Streams.h"
+
+
+static NSString *RUN_LOOP_MODE = @"HTTPConnectionRunLoopMode";
+
+@interface NSURLRequest (privateInterface)
++ (void)setAllowsAnyHTTPSCertificate:(BOOL)allow forHost:(NSString *)host;
+@end
+
+@interface URLConnection (internal)
+- (void)subtractBytes;
+@end
+
+@implementation URLConnection
++ (NSString *)errorDomain {
+    return @"URLConnectionErrorDomain";
+}
+
+- (id)initWithURL:(NSURL *)theURL method:(NSString *)theMethod delegate:(id)theDelegate {
+    if (self = [super init]) {
+        // Don't retain the delegate.
+        delegate = theDelegate;
+        method = [theMethod retain];
+        mutableURLRequest = [[NSMutableURLRequest alloc] initWithURL:theURL];
+        [mutableURLRequest setHTTPMethod:theMethod];
+        [NSURLRequest setAllowsAnyHTTPSCertificate:YES forHost:[theURL host]];
+        dateFormatter = [[RFC2616DateFormatter alloc] init];
+        HSLogTrace(@"%@ %@", theMethod, theURL);
+    }
+    return self;
+}
+- (void)dealloc {
+    [method release];
+    [urlConnection unscheduleFromRunLoop:[NSRunLoop currentRunLoop] forMode:RUN_LOOP_MODE];
+    [mutableURLRequest release];
+    [urlConnection release];
+    [httpURLResponse release];
+    [dateFormatter release];
+    [super dealloc];
+}
+- (void)setRequestHeader:(NSString *)value forKey:(NSString *)key {
+    HSLogTrace(@"request header %@ = %@", key, value);
+    [mutableURLRequest setValue:value forHTTPHeaderField:key];
+}
+- (void)setRequestHostHeader {
+    [self setRequestHeader:[[mutableURLRequest URL] host] forKey:@"Host"];
+}
+- (void)setRequestContentDispositionHeader:(NSString *)downloadName {
+    if (downloadName != nil) {
+        NSString *encodedFilename = [NSString stringWithFormat:@"\"%@\"", [downloadName stringByReplacingOccurrencesOfString:@"\"" withString:@"\\\\\""]];
+        encodedFilename = [encodedFilename stringByReplacingOccurrencesOfString:@"\n" withString:@"\\n"];
+        NSString *contentDisposition = [NSString stringWithFormat:@"attachment;filename=%@", encodedFilename];
+        [self setRequestHeader:contentDisposition forKey:@"Content-Disposition"];
+    }
+}
+- (void)setRFC822DateRequestHeader {
+    [self setRequestHeader:[dateFormatter rfc2616StringFromDate:[NSDate date]] forKey:@"Date"];
+}
+- (NSString *)requestMethod {
+    return [mutableURLRequest HTTPMethod];
+}
+- (NSString *)requestPathInfo {
+    return [[[mutableURLRequest URL] path] stringByAddingPercentEscapesUsingEncoding:NSUTF8StringEncoding];
+}
+- (NSString *)requestQueryString {
+    return [[mutableURLRequest URL] query];
+}
+- (NSArray *)requestHeaderKeys {
+    return [[mutableURLRequest allHTTPHeaderFields] allKeys];
+}
+- (NSString *)requestHeaderForKey:(NSString *)theKey {
+    return [[mutableURLRequest allHTTPHeaderFields] objectForKey:theKey];
+}
+- (BOOL)executeRequest:(NSError **)error {
+    return [self executeRequestWithBody:nil error:error];
+}
+- (BOOL)executeRequestWithBody:(NSData *)theBody error:(NSError **)error {
+    if (theBody != nil) {
+        [mutableURLRequest setHTTPBody:theBody];
+    }
+    totalSent = 0;
+    NSAssert(urlConnection == nil, @"can't call this method more than once!");
+    urlConnection = [[NSURLConnection alloc] initWithRequest:mutableURLRequest delegate:self startImmediately:NO];
+    [urlConnection scheduleInRunLoop:[NSRunLoop currentRunLoop] forMode:RUN_LOOP_MODE];
+    [urlConnection start];
+    return YES;
+}
+- (int)responseCode {
+    return [httpURLResponse statusCode];
+}
+- (NSString *)responseHeaderForKey:(NSString *)key {
+    return [[httpURLResponse allHeaderFields] objectForKey:key];
+}
+- (NSString *)responseContentType {
+    return [self responseHeaderForKey:@"Content-Type"];
+}
+- (NSString *)responseDownloadName {
+    NSString *downloadName = nil;
+    NSString *contentDisposition = [self responseHeaderForKey:@"Content-Disposition"];
+    if (contentDisposition != nil) {
+        NSRange filenameRange = [contentDisposition rangeOfRegex:@"attachment;filename=(.+)" capture:1];
+        if (filenameRange.location != NSNotFound) {
+            downloadName = [contentDisposition substringWithRange:filenameRange];
+        }
+    }
+    return downloadName;
+}
+- (id <InputStream>)newResponseBodyStream:(NSError **)error {
+    while (!complete && offset >= [receivedData length]) {
+        [[NSRunLoop currentRunLoop] runMode:RUN_LOOP_MODE beforeDate:[NSDate distantFuture]];
+    }
+    if (_error) {
+        if (error != NULL) {
+            *error = _error;
+        }
+        [self subtractBytes];
+        return nil;
+    }
+    
+    id <InputStream> ret = nil;
+    if ([method isEqualToString:@"HEAD"] && complete) {
+        HSLogTrace(@"%@: empty response body", self);
+        ret = [[NSData data] newInputStream];
+    } else {
+        NSAssert(httpURLResponse != nil, @"httpURLResponse can't be nil");
+        NSString *transferEncoding = [self responseHeaderForKey:@"Transfer-Encoding"];
+        HSLogTrace(@"Content-Length = %@", [self responseHeaderForKey:@"Content-Length"]);
+        HSLogTrace(@"Transfer-Encoding = %@", transferEncoding);
+        if (transferEncoding != nil && ![transferEncoding isEqualToString:@"Identity"]) {
+            if ([[transferEncoding lowercaseString] isEqualToString:@"chunked"]) {
+                HSLogTrace(@"%@: chunked response body", self);
+                id <InputStream> underlying = self;
+                BufferedInputStream *bis = [[BufferedInputStream alloc] initWithUnderlyingStream:underlying];
+                ret = [[ChunkedInputStream alloc] initWithUnderlyingStream:bis];
+                [bis release];
+            } else {
+                SETNSERROR(@"StreamErrorDomain", -1, @"unknown Transfer-Encoding '%@'", transferEncoding);
+                [self subtractBytes];
+                return nil;
+            }
+        } else {
+            /* 
+             * FIXME: handle multipart/byteranges media type.
+             * See rfc2616 section 4.4 ("message length").
+             */
+            HSLogTrace(@"%@: response body with no content-length", self);
+            ret = [self retain];
+        }
+    }
+    return ret;
+}    
+
+#pragma mark InputStream
+- (NSInteger)read:(unsigned char *)buf bufferLength:(NSUInteger)length error:(NSError **)error {
+    NSInteger recvd = 0;
+    while (!complete && offset >= [receivedData length]) {
+        [[NSRunLoop currentRunLoop] runMode:RUN_LOOP_MODE beforeDate:[NSDate distantFuture]];
+    }
+    if (_error) {
+        if (error != NULL) {
+            *error = _error;
+        }
+        [self subtractBytes];
+        return -1;
+    }
+    
+    NSUInteger bytesRemaining = [receivedData length] - offset;
+    recvd = (length < bytesRemaining) ? length : bytesRemaining;
+    totalReceived += recvd;
+    memcpy(buf, (unsigned char *)[receivedData bytes] + offset, recvd);
+    offset += recvd;
+    if (offset == [receivedData length]) {
+        [receivedData release];
+        receivedData = nil;
+        offset = 0;
+    }
+    HSLogTrace(@"received %d bytes", recvd);
+
+    return recvd;    
+}
+- (NSData *)slurp:(NSError **)error {
+    NSMutableData *ret = [NSMutableData data];
+    
+    if (receivedData != nil) {
+        [ret appendData:receivedData];
+        [receivedData release];
+        receivedData = nil;
+        offset = 0;
+    }
+    for (;;) {
+        while (!complete && offset >= [receivedData length]) {
+            [[NSRunLoop currentRunLoop] runMode:RUN_LOOP_MODE beforeDate:[NSDate distantFuture]];
+        }
+        if (_error) {
+            if (error != NULL) { *error = _error; }
+            return nil;
+        }
+        if (receivedData != nil) {
+            [ret appendData:receivedData];
+            [receivedData release];
+            receivedData = nil;
+        }
+        if (complete) {
+            break;
+        }
+    }
+    return ret;
+}
+
+#pragma mark NSURLConnection delegate
+- (void)connection:(NSURLConnection *)connection didFailWithError:(NSError *)myError {
+    HSLogDebug(@"%@ %@: %@", method, [mutableURLRequest URL], myError);
+    [_error release];
+    _error = [myError retain];
+    complete = YES;
+}
+- (void)connection:(NSURLConnection *)connection didReceiveData:(NSData *)data {
+    NSAssert(receivedData == nil, @"must not discard unread bytes");
+    HSLogTrace(@"received %u bytes", [data length]);
+    [receivedData release];
+    receivedData = [data retain];
+    offset = 0;
+}
+- (void)connection:(NSURLConnection *)connection didReceiveResponse:(NSURLResponse *)response {
+    if ([response isKindOfClass:[NSHTTPURLResponse class]]) {
+        [httpURLResponse release];
+        httpURLResponse = [response retain];
+    }
+}
+- (void)connection:(NSURLConnection *)connection didSendBodyData:(NSInteger)bytesWritten totalBytesWritten:(NSInteger)totalBytesWritten totalBytesExpectedToWrite:(NSInteger)totalBytesExpectedToWrite {
+    HSLogTrace(@"%@: sent so far %u of %u", self, totalBytesWritten, totalBytesExpectedToWrite);
+}
+- (NSCachedURLResponse *)connection:(NSURLConnection *)connection willCacheResponse:(NSCachedURLResponse *)cachedResponse {
+    return cachedResponse;
+}
+- (NSURLRequest *)connection:(NSURLConnection *)connection willSendRequest:(NSURLRequest *)request redirectResponse:(NSURLResponse *)redirectResponse {
+    return request;
+}
+- (void)connectionDidFinishLoading:(NSURLConnection *)connection {
+    complete = YES;
+}
+
+#pragma mark NSObject
+- (NSString *)description {
+    return [NSString stringWithFormat:@"<URLConnection: %@>", [mutableURLRequest URL]];
+}
+@end
+
+@implementation URLConnection (internal)
+- (void)subtractBytes {
+    if (totalSent > 0 && [delegate respondsToSelector:@selector(urlConnection:subtractSentBytes:)]) {
+        [delegate urlConnection:self subtractSentBytes:totalSent];
+        totalSent = 0;
+    }
+}
+@end

io/BooleanIO.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -31,13 +31,12 @@
  */ 
 
 #import <Cocoa/Cocoa.h>
-#import "BufferedInputStream.h"
+@class BufferedInputStream;
-#import "OutputStream.h"
+@class BufferedOutputStream;
-
 @interface BooleanIO : NSObject {
 
 }
 + (void)write:(BOOL)b to:(NSMutableData *)data;
-+ (BOOL)write:(BOOL)b to:(id <OutputStream>)os error:(NSError **)error;
++ (BOOL)write:(BOOL)b to:(BufferedOutputStream *)os error:(NSError **)error;
 + (BOOL)read:(BOOL *)value from:(BufferedInputStream *)is error:(NSError **)error;
 @end

io/BooleanIO.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -31,18 +31,17 @@
  */ 
 
 #import "BooleanIO.h"
-#import "Streams.h"
 #import "BufferedInputStream.h"
+#import "BufferedOutputStream.h"
 
 @implementation BooleanIO
 + (void)write:(BOOL)b to:(NSMutableData *)data {
     char c = b ? 1 : 0;
     [data appendBytes:&c length:1];
 }
-+ (BOOL)write:(BOOL)b to:(id <OutputStream>)os error:(NSError **)error {
++ (BOOL)write:(BOOL)b to:(BufferedOutputStream *)os error:(NSError **)error {
     unsigned char c = b ? 1 : 0;
-    return [os write:&c length:1 error:error];
+    return [os writeFully:&c length:1 error:error];
-    
 }
 + (BOOL)read:(BOOL *)value from:(BufferedInputStream *)is error:(NSError **)error {
     *value = NO;

io/BufferedInputStream.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -44,5 +44,7 @@
 - (id)initWithUnderlyingStream:(id <InputStream>)theUnderlyingStream;
 - (NSData *)readExactly:(NSUInteger)exactLength error:(NSError **)error;
 - (BOOL)readExactly:(NSUInteger)exactLength into:(unsigned char *)buf error:(NSError **)error;
+- (NSString *)readLineWithCRLFWithMaxLength:(NSUInteger)maxLength error:(NSError **)error;
+- (NSString *)readLine:(NSError **)error;
 - (uint64_t)bytesReceived;
 @end

io/BufferedInputStream.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -36,11 +36,11 @@
 #import "InputStreams.h"
 #import "SetNSError.h"
 
-#define MY_BUF_SIZE (8192)
+#define MY_BUF_SIZE (4096)
 
 @implementation BufferedInputStream
 + (NSString *)errorDomain {
-    return @"BufInputStreamErrorDomain";
+    return @"BufferedInputStreamErrorDomain";
 }
 - (id)initWithUnderlyingStream:(id <InputStream>)theUnderlyingStream {
     if (self = [super init]) {
@@ -76,14 +76,54 @@
             return NO;
         }
         if (ret == 0) {
-            SETNSERROR([BufferedInputStream errorDomain], ERROR_EOF, @"EOF after %u of %u bytes received", received, exactLength);
+            SETNSERROR([BufferedInputStream errorDomain], ERROR_EOF, @"%@ EOF after %u of %u bytes received", self, received, exactLength);
             return NO;
         }
         received += ret;
-        totalBytesReceived += ret;
     }
     return YES;
 }
+- (NSString *)readLineWithCRLFWithMaxLength:(NSUInteger)maxLength error:(NSError **)error {
+    unsigned char *lineBuf = (unsigned char *)malloc(maxLength);
+    NSUInteger received = 0;
+    for (;;) {
+        if (received > maxLength) {
+            SETNSERROR(@"InputStreamErrorDomain", -1, @"exceeded maxLength %u before finding CRLF", maxLength);
+            free(lineBuf);
+            return nil;
+        }
+        if (![self readExactly:1 into:(lineBuf + received) error:error]) {
+            free(lineBuf);
+            return nil;
+        }
+        received++;
+        if (received >= 2 && lineBuf[received - 1] == '\n' && lineBuf[received - 2] == '\r') {
+            break;
+        }
+    }
+    NSString *ret = [[[NSString alloc] initWithBytes:lineBuf length:received encoding:NSUTF8StringEncoding] autorelease];
+    free(lineBuf);
+    HSLogTrace(@"got line <%@>", ret);
+    return ret;
+}
+- (NSString *)readLine:(NSError **)error {
+    NSMutableData *data = [NSMutableData data];
+    unsigned char charBuf[1];
+    NSUInteger received = 0;
+    for (;;) {
+        if (![self readExactly:1 into:charBuf error:error]) {
+            return nil;
+        }
+        if (*charBuf == '\n') {
+            break;
+        }
+        [data appendBytes:charBuf length:1];
+        received++;
+    }
+    NSString *ret = [[[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] autorelease];
+    HSLogTrace(@"got line <%@> followed by \\n", ret);
+    return ret;
+}
 - (uint64_t)bytesReceived {
     return totalBytesReceived;
 }
@@ -110,15 +150,23 @@
         len = myRet;
         if (len > 0) {
             ret = len > outBufLen ? outBufLen : len;
-            memcpy(outBuf, buf + pos, ret);
+            memcpy(outBuf, buf, ret);
             pos += ret;
        } else {
            ret = 0;
        }
     }
+    if (ret > 0) {
+        totalBytesReceived += ret;
+    }
     return ret;
 }
 - (NSData *)slurp:(NSError **)error {
     return [InputStreams slurp:self error:error];
 }
+
+#pragma mark NSObject
+- (NSString *)description {
+    return [NSString stringWithFormat:@"<BufferedInputStream %@>", underlyingStream];
+}
 @end

io/BufferedOutputStream.h

@@ -0,0 +1,29 @@
+//
+//  BufferedOutputStream.h
+//  iPhotoSync
+//
+//  Created by Stefan Reitshamer on 8/25/10.
+//  Copyright 2010 __MyCompanyName__. All rights reserved.
+//
+
+#import <Cocoa/Cocoa.h>
+#import "OutputStream.h"
+
+@interface BufferedOutputStream : NSObject <OutputStream> {
+    id <OutputStream> os;
+    unsigned char *buf;
+    NSUInteger pos;
+    NSUInteger buflen;
+    uint64_t totalBytesWritten;
+    BOOL errorOccurred;
+}
++ (NSString *)errorDomain;
+- (id)initWithMutableData:(NSMutableData *)theMutableData;
+- (id)initWithFD:(int)theFD;
+- (id)initWithPath:(NSString *)thePath append:(BOOL)isAppend;
+- (id)initWithPath:(NSString *)thePath targetUID:(uid_t)theTargetUID targetGID:(gid_t)theTargetGID append:(BOOL)isAppend;
+- (id)initWithUnderlyingOutputStream:(id <OutputStream>)theOS;
+- (BOOL)setBufferSize:(NSUInteger)size error:(NSError **)error;
+- (BOOL)writeFully:(const unsigned char *)buf length:(NSUInteger)len error:(NSError **)error;
+- (BOOL)flush:(NSError **)error;
+@end

io/BufferedOutputStream.m

@@ -0,0 +1,142 @@
+//
+//  BufferedOutputStream.m
+//  iPhotoSync
+//
+//  Created by Stefan Reitshamer on 8/25/10.
+//  Copyright 2010 __MyCompanyName__. All rights reserved.
+//
+
+#import "BufferedOutputStream.h"
+#import "NSErrorCodes.h"
+#import "SetNSError.h"
+#import "DataOutputStream.h"
+#import "FDOutputStream.h"
+#import "FileOutputStream.h"
+
+#define MY_BUF_SIZE (4096)
+
+@implementation BufferedOutputStream
++ (NSString *)errorDomain {
+    return @"BufferedOutputStreamErrorDomain";
+}
+- (id)initWithMutableData:(NSMutableData *)theMutableData {
+    DataOutputStream *dos = [[DataOutputStream alloc] initWithMutableData:theMutableData];
+    id ret = [self initWithUnderlyingOutputStream:dos];
+    [dos release];
+    return ret;
+}
+- (id)initWithFD:(int)theFD {
+    FDOutputStream *fdos = [[FDOutputStream alloc] initWithFD:theFD];
+    id ret = [self initWithUnderlyingOutputStream:fdos];
+    [fdos release];
+    return ret;
+}
+- (id)initWithPath:(NSString *)thePath append:(BOOL)isAppend {
+    FileOutputStream *fos = [[FileOutputStream alloc] initWithPath:thePath append:isAppend];
+    id ret = [self initWithUnderlyingOutputStream:fos];
+    [fos release];
+    return ret;
+}
+- (id)initWithPath:(NSString *)thePath targetUID:(uid_t)theTargetUID targetGID:(gid_t)theTargetGID append:(BOOL)isAppend {
+    FileOutputStream *fos = [[FileOutputStream alloc] initWithPath:thePath targetUID:theTargetUID targetGID:theTargetGID append:isAppend];
+    id ret = [self initWithUnderlyingOutputStream:fos];
+    [fos release];
+    return ret;
+}
+- (id)initWithUnderlyingOutputStream:(id <OutputStream>)theOS {
+    if (self = [super init]) {
+        os = [theOS retain];
+        buflen = MY_BUF_SIZE;
+        buf = (unsigned char *)malloc(buflen);
+    }
+    return self;
+}
+- (void)dealloc {
+    if (pos > 0 && !errorOccurred) {
+        HSLogWarn(@"BufferedOutputStream pos > 0 -- flush wasn't called?!");
+    }
+    [os release];
+    free(buf);
+    [super dealloc];
+}
+- (BOOL)setBufferSize:(NSUInteger)size error:(NSError **)error {
+    if (![self flush:error]) {
+        return NO;
+    }
+    buf = realloc(buf, size);
+    buflen = size;
+    return YES;
+}
+- (BOOL)flush:(NSError **)error {
+    NSAssert(os != nil, @"write: os can't be nil");
+    NSUInteger index = 0;
+    while (index < pos) {
+        NSInteger written = [os write:&buf[index] length:(pos - index) error:error];
+        if (written < 0) {
+            errorOccurred = YES;
+            return NO;
+        }
+        if (written == 0) {
+            SETNSERROR([BufferedOutputStream errorDomain], ERROR_EOF, @"0 bytes written to underlying stream %@", [os description]);
+            errorOccurred = YES;
+            return NO;
+        }
+        index += written;
+    }
+    pos = 0;
+    return YES;
+}
+- (BOOL)writeFully:(const unsigned char *)theBuf length:(NSUInteger)len error:(NSError **)error {
+    NSUInteger totalWritten = 0;
+    while (totalWritten < len) {
+        NSInteger writtenThisTime = [self write:&theBuf[totalWritten] length:(len - totalWritten) error:error];
+        if (writtenThisTime < 0) {
+            return NO;
+        }
+        totalWritten += (NSUInteger)writtenThisTime;
+    }
+    NSAssert(totalWritten == len, @"writeFully must return as all bytes written");
+    return YES;
+}
+
+#pragma mark OutputStream
+- (NSInteger)write:(const unsigned char *)theBuf length:(NSUInteger)theLen error:(NSError **)error {
+    NSAssert(os != nil, @"write: os can't be nil");
+    if ((pos + theLen) > buflen) {
+        if (![self flush:error]) {
+            errorOccurred = YES;
+            return -1;
+        }
+    }
+    if (theLen > buflen) {
+        NSUInteger written = 0;
+        // Loop to write theBuf directly to the underlying stream, since it won't fit in our buffer.
+        while (written < theLen) {
+            NSInteger writtenThisTime = [os write:&theBuf[written] length:(theLen - written) error:error];
+            if (writtenThisTime < 0) {
+                errorOccurred = YES;
+                return -1;
+            }
+            if (writtenThisTime == 0) {
+                SETNSERROR([BufferedOutputStream errorDomain], ERROR_EOF, @"0 bytes written to underlying stream");
+                errorOccurred = YES;
+                return -1;
+            }
+            written += writtenThisTime;
+        }
+    } else {
+        memcpy(buf + pos, theBuf, theLen);
+        pos += theLen;
+    }
+    totalBytesWritten += theLen;
+    return theLen;
+}
+- (unsigned long long)bytesWritten {
+    return totalBytesWritten;
+}
+
+#pragma mark NSObject
+- (NSString *)description {
+    return [NSString stringWithFormat:@"<BufferedOutputStream underlying=%@>", os];
+}
+@end

io/CFStreamInputStream.m

@@ -1,102 +0,0 @@
-/*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
-
-#import "CFStreamInputStream.h"
-#import "InputStreams.h"
-#import "SetNSError.h"
-#import "NSErrorCodes.h"
-#import "CFStreamPair.h"
-
-#define MY_BUF_SIZE (8192)
-#define DEFAULT_READ_TIMEOUT_SECONDS (60)
-
-@interface CFStreamInputStream (internal)
-- (BOOL)open:(NSError **)error;
-@end
-
-@implementation CFStreamInputStream
-- (id)initWithCFReadStream:(CFReadStreamRef)streamRef {
-    if (self = [super init]) {
-        readStream = streamRef;
-        CFRetain(readStream);
-    }
-    return self;
-}
-- (void)dealloc {
-    CFRelease(readStream);
-    [super dealloc];
-}
-
-#pragma mark InputStream
-- (NSInteger)read:(unsigned char *)buf bufferLength:(NSUInteger)bufferLength error:(NSError **)error {
-    if (![self open:error]) {
-        return -1;
-    }
-    CFIndex index = CFReadStreamRead(readStream, buf, bufferLength);
-    if (index == -1) {
-        if (error != NULL) {
-            CFErrorRef err = CFReadStreamCopyError(readStream);
-            if (err == NULL) {
-                SETNSERROR(@"CFStreamPairErrorDomain", -1, @"unknown network error");
-            } else {
-                *error = [CFStreamPair NSErrorWithNetworkError:err];
-                CFRelease(err);
-            }
-        }
-        return -1;
-    }
-    return (NSInteger)index;
-}
-- (NSData *)slurp:(NSError **)error {
-    return [InputStreams slurp:self error:error];
-}
-@end
-@implementation CFStreamInputStream (internal)
-- (BOOL)open:(NSError **)error {
-    if (!isOpen) {
-        if (!CFReadStreamOpen(readStream)) {
-            if (error != NULL) {
-                CFErrorRef err = CFReadStreamCopyError(readStream);
-                if (err == NULL) {
-                    SETNSERROR(@"CFStreamPairErrorDomain", -1, @"unknown network error");
-                } else {
-                    *error = [CFStreamPair NSErrorWithNetworkError:err];
-                    CFRelease(err);
-                }
-            }
-            return NO;
-        }
-        isOpen = YES;
-    }
-    return YES;
-}
-@end

io/CFStreamOutputStream.m

@@ -1,86 +0,0 @@
-/*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of 
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */ 
-
-#import "CFStreamOutputStream.h"
-#import "SetNSError.h"
-#import "CFStreamPair.h"
-
-@implementation CFStreamOutputStream
-- (id)initWithCFWriteStream:(CFWriteStreamRef)streamRef {
-    if (self = [super init]) {
-        writeStream = streamRef;
-        CFRetain(streamRef);
-    }
-    return self;
-}
-- (void)dealloc {
-    CFWriteStreamClose(writeStream);
-    CFRelease(writeStream);
-    [super dealloc];
-}
-
-#pragma mark OutputStream
-- (BOOL)write:(const unsigned char *)buf length:(NSUInteger)len error:(NSError **)error {
-    if (!isOpen) {
-        Boolean ret = CFWriteStreamOpen(writeStream);
-        if (ret == false) {
-            SETNSERROR(@"CFStreamErrorDomain", -1, @"error opening write stream");
-            return NO;
-        }
-        isOpen = YES;
-    }
-    CFIndex index = 0;
-    NSUInteger written = 0;
-    while ((len - written) > 0) {
-    write_again:
-        index = CFWriteStreamWrite(writeStream, &(buf[written]), len - written);
-        if (index == -1) {
-            if (error != NULL) {
-                CFErrorRef err = CFWriteStreamCopyError(writeStream);
-                if (err == NULL) {
-                    SETNSERROR(@"CFStreamPairErrorDomain", -1, @"unknown network error");
-                } else {
-                    *error = [CFStreamPair NSErrorWithNetworkError:err];
-                    CFRelease(err);
-                }
-            }
-            return NO;
-        }
-        written += (unsigned long long)index;
-        bytesWritten += (unsigned long long)index;
-    }
-    return YES;
-}
-- (unsigned long long)bytesWritten {
-    return bytesWritten;
-}
-@end

io/ChunkedInputStream.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  

io/ChunkedInputStream.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -54,19 +54,22 @@
 - (NSInteger)read:(unsigned char *)buf bufferLength:(NSUInteger)bufferLength error:(NSError **)error {
     if (received >= chunkLength) {
         received = 0;
-        NSString *line = [InputStreams readLineWithCRLF:underlyingStream maxLength:MAX_CHUNK_LENGTH_LINE_LENGTH error:error];
+        NSString *line = [underlyingStream readLineWithCRLFWithMaxLength:MAX_CHUNK_LENGTH_LINE_LENGTH error:error];
         if (line == nil) {
             return -1;
         }
         NSScanner *scanner = [NSScanner scannerWithString:line];
-        if (![scanner scanHexInt:&chunkLength]) {
+        unsigned int scanned = 0;
+        if (![scanner scanHexInt:&scanned]) {
             SETNSERROR(@"StreamErrorDomain", -1, @"invalid chunk length: %@", line);
             return -1;
         }
+        chunkLength = (NSUInteger)scanned;
         HSLogTrace(@"chunk length = %u", chunkLength);
     }
     if (chunkLength == 0) {
-        return 0;
+        SETNSERROR(@"StreamsErrorDomain", ERROR_EOF, @"EOF (zero chunk length)");
+        return -1;
     }
     NSUInteger remaining = chunkLength - received;
     NSUInteger toRead = remaining > bufferLength ? bufferLength : remaining;
@@ -76,7 +79,7 @@
     }
     received += ret;
     if (received >= chunkLength) {
-        NSString *line = [InputStreams readLineWithCRLF:underlyingStream maxLength:MAX_CHUNK_LENGTH_LINE_LENGTH error:error];
+        NSString *line = [underlyingStream readLineWithCRLFWithMaxLength:MAX_CHUNK_LENGTH_LINE_LENGTH error:error];
         if (line == nil) {
             return -1;
         }

io/CryptInputStream.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -33,6 +33,7 @@
 #import <Cocoa/Cocoa.h>
 #include <openssl/evp.h>
 #import "InputStream.h"
+@class CryptoKey;
 
 typedef int (*CryptInitFunc)(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type, unsigned char *key, unsigned char *iv);
 typedef int (*CryptUpdateFunc)(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl, unsigned char *in, int inl);
@@ -43,20 +44,23 @@ typedef int (*CryptFinalFunc)(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl
     CryptUpdateFunc cryptUpdate;
     CryptFinalFunc cryptFinal;
     id <InputStream> is;
+    NSString *label;
     unsigned char *inBuf;
     NSUInteger inBufSize;
     unsigned char *outBuf;
     NSInteger outBufLen;
     NSUInteger outBufSize;
     NSUInteger outBufPos;
-    const EVP_CIPHER *cipher;
     EVP_CIPHER_CTX cipherContext;
-    unsigned char evp_key[EVP_MAX_KEY_LENGTH];
-    unsigned char iv[EVP_MAX_IV_LENGTH];
     size_t blockSize;
     BOOL initialized;
     BOOL finalized;
 }
-+ (NSString *)errorDomain;
+- (id)initWithCryptInitFunc:(void *)theCryptInit 
-- (id)initWithCryptInitFunc:(void *)theCryptInit cryptUpdateFunc:(void *)theCryptUpdate cryptFinalFunc:(void *)theCryptFinal inputStream:(id <InputStream>)theIS cipherName:(NSString *)theCipherName key:(NSString *)theKey error:(NSError **)error;
+            cryptUpdateFunc:(void *)theCryptUpdate 
+             cryptFinalFunc:(void *)theCryptFinal 
+                inputStream:(id <InputStream>)theIS 
+                  cryptoKey:(CryptoKey *)theCryptoKey
+                      label:(NSString *)theLabel
+                      error:(NSError **)error;
 @end

io/CryptInputStream.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -30,48 +30,41 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */ 
 
+#import <Cocoa/Cocoa.h>
+
 #import "CryptInputStream.h"
 #import "SetNSError.h"
 #import "OpenSSL.h"
 #import "InputStreams.h"
 #import "NSErrorCodes.h"
+#import "CryptoKey.h"
+#import "Encryption.h"
 
-#define MY_BUF_SIZE (8192)
+#define MY_BUF_SIZE (4096)
 
 @interface CryptInputStream (internal)
 - (BOOL)fillOutBuf:(NSError **)error;
 @end
 
 @implementation CryptInputStream
-+ (NSString *)errorDomain {
+- (id)initWithCryptInitFunc:(void *)theCryptInit 
-    return @"CryptInputStreamErrorDomain";
+            cryptUpdateFunc:(void *)theCryptUpdate 
-}
+             cryptFinalFunc:(void *)theCryptFinal 
-- (id)initWithCryptInitFunc:(void *)theCryptInit cryptUpdateFunc:(void *)theCryptUpdate cryptFinalFunc:(void *)theCryptFinal inputStream:(id <InputStream>)theIS cipherName:(NSString *)theCipherName key:(NSString *)theKey error:(NSError **)error {
+                inputStream:(id <InputStream>)theIS 
+                  cryptoKey:(CryptoKey *)theCryptoKey
+                      label:(NSString *)theLabel
+                      error:(NSError **)error {
     if (self = [super init]) {
+        label = [theLabel retain];
         cryptInit = (CryptInitFunc)theCryptInit;
         cryptUpdate = (CryptUpdateFunc)theCryptUpdate;
         cryptFinal = (CryptFinalFunc)theCryptFinal;
         BOOL ret = NO;
         do {
             is = [theIS retain];
-            NSData *keyData = [theKey dataUsingEncoding:NSUTF8StringEncoding];
-            if ([keyData length] > EVP_MAX_KEY_LENGTH) {
-                SETNSERROR([CryptInputStream errorDomain], -1, @"encryption key must be less than or equal to %d bytes", EVP_MAX_KEY_LENGTH);
-                break;
-            }
-            if (![OpenSSL initializeSSL:error]) {
-                break;
-            }
-            cipher = EVP_get_cipherbyname([theCipherName UTF8String]);
-            if (!cipher) {
-                SETNSERROR([CryptInputStream errorDomain], -1, @"failed to load %@ cipher: %@", theCipherName, [OpenSSL errorMessage]);
-                break;
-            }
-            evp_key[0] = 0;
-            EVP_BytesToKey(cipher, EVP_md5(), NULL, [keyData bytes], [keyData length], 1, evp_key, iv);
             EVP_CIPHER_CTX_init(&cipherContext);
-            if (!(*cryptInit)(&cipherContext, cipher, evp_key, iv)) {
+            if (!(*cryptInit)(&cipherContext, [theCryptoKey cipher], [theCryptoKey evpKey], [theCryptoKey iv])) {
-                SETNSERROR([CryptInputStream errorDomain], -1, @"EVP_EncryptInit: %@",  [OpenSSL errorMessage]);
+                SETNSERROR([Encryption errorDomain], -1, @"%@ initialization error: %@",  label, [OpenSSL errorMessage]);
                 break;
             }
             EVP_CIPHER_CTX_set_key_length(&cipherContext, EVP_MAX_KEY_LENGTH);
@@ -91,6 +84,7 @@
     return self;
 }
 - (void)dealloc {
+    [label release];
     if (initialized) {
         EVP_CIPHER_CTX_cleanup(&cipherContext);
     }
@@ -135,7 +129,7 @@
 @implementation CryptInputStream (internal)
 - (BOOL)fillOutBuf:(NSError **)error {
     if (finalized) {
-        SETNSERROR([CryptInputStream errorDomain], ERROR_EOF, @"EOF");
+        SETNSERROR([Encryption errorDomain], ERROR_EOF, @"EOF");
         return NO;
     }
     outBufLen = 0;
@@ -146,15 +140,21 @@
     }
     if (recvd == 0) {
         finalized = YES;
-        if (!(cryptFinal)(&cipherContext, outBuf, &outBufLen)) {
+        int theBufLen = 0;
-            SETNSERROR(@"OpenSSLErrorDomain", -1, @"crypt final: %@", [OpenSSL errorMessage]);
+        if (!(cryptFinal)(&cipherContext, outBuf, &theBufLen)) {
+            SETNSERROR([Encryption errorDomain], -1, @"%@ error: %@", label, [OpenSSL errorMessage]);
             return NO;
         }
+        HSLogTrace(@"%@ final: outBufLen = %d", label, (NSInteger)outBufLen);
+        outBufLen = (NSInteger)theBufLen;
     } else {
-        if (!(*cryptUpdate)(&cipherContext, outBuf, &outBufLen, inBuf, recvd)) {
+        int theBufLen = 0;
-            SETNSERROR(@"OpenSSLErrorDomain", -1, @"crypt update: %@", [OpenSSL errorMessage]);
+        if (!(*cryptUpdate)(&cipherContext, outBuf, &theBufLen, inBuf, recvd)) {
+            SETNSERROR([Encryption errorDomain], -1, @"%@ error: %@", label, [OpenSSL errorMessage]);
             return NO;
         }
+        HSLogTrace(@"%@ update: inBufLen = %d, outBufLen = %d", label, recvd, (NSInteger)outBufLen);
+        outBufLen = (NSInteger)theBufLen;
     }
     return YES;
 }

io/CryptoKey.h

@@ -0,0 +1,25 @@
+//
+//  CryptoKey.h
+//  Arq
+//
+//  Created by Stefan Reitshamer on 6/9/11.
+//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#import <Cocoa/Cocoa.h>
+#include <openssl/evp.h>
+
+
+@interface CryptoKey : NSObject {
+    const EVP_CIPHER *cipher;
+    unsigned char evpKey[EVP_MAX_KEY_LENGTH];
+    unsigned char iv[EVP_MAX_IV_LENGTH];
+}
++ (NSString *)errorDomain;
+
+- (id)initWithPassword:(NSString *)thePassword salt:(NSData *)theSalt error:(NSError **)error;
+- (id)initLegacyWithPassword:(NSString *)thePassword error:(NSError **)error;
+- (EVP_CIPHER *)cipher;
+- (unsigned char *)evpKey;
+- (unsigned char *)iv;
+@end

io/CryptoKey.m

@@ -0,0 +1,79 @@
+//
+//  CryptoKey.m
+//  Arq
+//
+//  Created by Stefan Reitshamer on 6/9/11.
+//  Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#import "CryptoKey.h"
+#import "SetNSError.h"
+#import "OpenSSL.h"
+#import "Encryption.h"
+
+#define ITERATIONS (1000)
+#define KEYLEN (48)
+
+@implementation CryptoKey
++ (NSString *)errorDomain {
+    return @"CryptoKeyErrorDomain";
+}
+
+- (id)init {
+    @throw [NSException exceptionWithName:@"InvalidInitializerException" reason:@"can't call CryptoKey init" userInfo:nil];
+}
+- (id)initWithPassword:(NSString *)thePassword salt:(NSData *)theSalt error:(NSError **)error {
+    if (self = [super init]) {
+        if (![OpenSSL initializeSSL:error]) {
+            [self release];
+            return nil;
+        }
+        if (theSalt != nil && [theSalt length] != 8) {
+            SETNSERROR([Encryption errorDomain], -1, @"salt must be 8 bytes or nil");
+            [self release];
+            return nil;
+        }
+        cipher = EVP_aes_256_cbc();
+        const char *cPassword = [thePassword UTF8String];
+        unsigned char *cSaltCopy = NULL;
+        if (theSalt != nil) {
+            cSaltCopy = (unsigned char *)malloc([theSalt length]);
+            memcpy(cSaltCopy, [theSalt bytes], [theSalt length]);
+        } else {
+            HSLogWarn(@"NULL salt value for CryptoKey");
+        }
+        unsigned char buf[KEYLEN];
+        memset(buf, 0, KEYLEN);
+        PKCS5_PBKDF2_HMAC_SHA1(cPassword, strlen(cPassword), cSaltCopy, [theSalt length], ITERATIONS, KEYLEN, buf);
+        evpKey[0] = 0;
+        EVP_BytesToKey(cipher, EVP_sha1(), cSaltCopy, buf, KEYLEN, ITERATIONS, evpKey, iv);
+        if (cSaltCopy != NULL) {
+            free(cSaltCopy);
+        }
+    }
+    return self;
+}
+- (id)initLegacyWithPassword:(NSString *)thePassword error:(NSError **)error {
+    if (self = [super init]) {
+        if (![OpenSSL initializeSSL:error]) {
+            [self release];
+            return nil;
+        }
+        
+        cipher = EVP_aes_256_cbc();
+        evpKey[0] = 0;
+        NSData *passwordData = [thePassword dataUsingEncoding:NSUTF8StringEncoding];        
+        EVP_BytesToKey(cipher, EVP_md5(), NULL, [passwordData bytes], [passwordData length], 1, evpKey, iv);
+    }
+    return self;
+}
+- (const EVP_CIPHER *)cipher {
+    return cipher;
+}
+- (unsigned char *)evpKey {
+    return evpKey;
+}
+- (unsigned char *)iv {
+    return iv;
+}
+@end

io/DataIO.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -19,7 +19,7 @@
  
  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FORfile://localhost/Users/stefan/src/arq/Arq/BackupVolumeCommitter.h
  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
@@ -30,15 +30,15 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */ 
 
-#import <Cocoa/Cocoa.h>
-
 #import <Cocoa/Cocoa.h>
 @class BufferedInputStream;
+@class BufferedOutputStream;
 
 @interface DataIO : NSObject {
 
 }
 + (void)write:(NSData *)data to:(NSMutableData *)data;
++ (BOOL)write:(NSData *)data to:(BufferedOutputStream *)os error:(NSError **)error;
 + (BOOL)read:(NSData **)value from:(BufferedInputStream *)is error:(NSError **)error;
 
 @end

io/DataIO.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -30,18 +30,20 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */ 
 
-#import <Cocoa/Cocoa.h>
-
 #import "DataIO.h"
 #import "IntegerIO.h"
-#import "Streams.h"
 #import "BufferedInputStream.h"
+#import "BufferedOutputStream.h"
 
 @implementation DataIO
 + (void)write:(NSData *)data to:(NSMutableData *)outData {
     [IntegerIO writeUInt64:(uint64_t)[data length] to:outData];
     [outData appendBytes:[data bytes] length:[data length]];
 }
++ (BOOL)write:(NSData *)data to:(BufferedOutputStream *)os error:(NSError **)error {
+    return ([IntegerIO writeUInt64:[data length] to:os error:error] && [os writeFully:[data bytes] length:[data length] error:error]);
+}
+
 + (BOOL)read:(NSData **)value from:(BufferedInputStream *)is error:(NSError **)error {
     *value = nil;
     uint64_t length = 0;

io/DataInputStream.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  

io/DataInputStream.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  

io/DataInputStreamFactory.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -30,11 +30,14 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */ 
 
+#import <Cocoa/Cocoa.h>
+
 #import <Cocoa/Cocoa.h>
 #import "InputStreamFactory.h"
 
 @interface DataInputStreamFactory : NSObject <InputStreamFactory> {
     NSData *data;
+    NSString *dataDescription;
 }
-- (id)initWithData:(NSData *)theData;
+- (id)initWithData:(NSData *)theData dataDescription:(NSString *)theDataDescription;
 @end

io/DataInputStreamFactory.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -37,14 +37,16 @@
 #import "NSData-InputStream.h"
 
 @implementation DataInputStreamFactory
-- (id)initWithData:(NSData *)theData {
+- (id)initWithData:(NSData *)theData dataDescription:(NSString *)theDataDescription {
     if (self = [super init]) {
         data = [theData retain];
+        dataDescription = [theDataDescription retain];
     }
     return self;
 }
 - (void)dealloc {
     [data release];
+    [dataDescription release];
     [super dealloc];
 }
 
@@ -55,6 +57,6 @@
 
 #pragma mark NSObject
 - (NSString *)description {
-    return [NSString stringWithFormat:@"<DataISF: %u>", [data length]];
+    return [NSString stringWithFormat:@"<DataISF: %u bytes: %@>", [data length], dataDescription];
 }
 @end

io/DataOutputStream.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -30,13 +30,14 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */ 
 
+#import <Cocoa/Cocoa.h>
+
 #import <Cocoa/Cocoa.h>
 #import "OutputStream.h"
 
-@interface CFStreamOutputStream : NSObject <OutputStream> {
+@interface DataOutputStream : NSObject <OutputStream> {
-    CFWriteStreamRef writeStream;
+    NSMutableData *mutableData;
-    BOOL isOpen;
     unsigned long long bytesWritten;
 }
-- (id)initWithCFWriteStream:(CFWriteStreamRef)streamRef;
+- (id)initWithMutableData:(NSMutableData *)theMutableData;
 @end

io/DataOutputStream.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -30,26 +30,34 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */ 
 
-#import "ArqFolder.h"
+#import <Cocoa/Cocoa.h>
-#import "DictNode.h"
 
-@implementation ArqFolder
+#import "DataOutputStream.h"
-- (id)initWithS3Path:(NSString *)theS3Path plist:(DictNode *)plist {
+
+
+@implementation DataOutputStream
+- (id)initWithMutableData:(NSMutableData *)theMutableData {
     if (self = [super init]) {
-        s3Path = [theS3Path copy];
+        mutableData = [theMutableData retain];
-        localPath = [[[plist stringNodeForKey:@"LocalPath"] stringValue] copy];
     }
     return self;
 }
 - (void)dealloc {
-    [s3Path release];
+    [mutableData release];
-    [localPath release];
     [super dealloc];
 }
-- (NSString *)s3Path {
+#pragma mark OutputStream protocol
-    return s3Path;
+- (NSInteger)write:(const unsigned char *)buf length:(NSUInteger)len error:(NSError **)error {
+    [mutableData appendBytes:buf length:len];
+    bytesWritten += len;
+    return len;
 }
-- (NSString *)localPath {
+- (unsigned long long)bytesWritten {
-    return localPath;
+    return bytesWritten;
+}
+
+#pragma mark NSObject
+- (NSString *)description {
+    return @"<DataOutputStream>";
 }
 @end

io/DateIO.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -32,10 +32,12 @@
 
 #import <Cocoa/Cocoa.h>
 @class BufferedInputStream;
+@class BufferedOutputStream;
 
 @interface DateIO : NSObject {
 
 }
 + (void)write:(NSDate *)date to:(NSMutableData *)data;
++ (BOOL)write:(NSDate *)date to:(BufferedOutputStream *)bos error:(NSError **)error;
 + (BOOL)read:(NSDate **)date from:(BufferedInputStream *)is error:(NSError **)error;
 @end

io/DateIO.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -30,8 +30,6 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */ 
 
-#import <Cocoa/Cocoa.h>
-
 #import "BooleanIO.h"
 #import "IntegerIO.h"
 #import "DateIO.h"
@@ -46,6 +44,19 @@
 		[IntegerIO writeInt64:millisecondsSince1970 to:data];
 	}
 }
++ (BOOL)write:(NSDate *)date to:(BufferedOutputStream *)bos error:(NSError **)error {
+	BOOL dateNotNil = (date != nil);
+    if (![BooleanIO write:dateNotNil to:bos error:error]) {
+        return NO;
+    }
+    if (dateNotNil) {
+		long long millisecondsSince1970 = (long long)([date timeIntervalSince1970] * 1000.0);
+        if (![IntegerIO writeInt64:millisecondsSince1970 to:bos error:error]) {
+            return NO;
+        }
+    }
+    return YES;
+}
 + (BOOL)read:(NSDate **)date from:(BufferedInputStream *)is error:(NSError **)error {
     *date = nil;
     BOOL notNil;

io/DecryptedInputStream.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -30,10 +30,13 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */ 
 
+#import <Cocoa/Cocoa.h>
+
 #import <Cocoa/Cocoa.h>
 #import "CryptInputStream.h"
+@class CryptoKey;
 
 @interface DecryptedInputStream : CryptInputStream {
 }
-- (id)initWithInputStream:(id <InputStream>)theIS cipherName:(NSString *)theCipherName key:(NSString *)theKey error:(NSError **)error;
+- (id)initWithInputStream:(id <InputStream>)theIS cryptoKey:(CryptoKey *)theCryptoKey error:(NSError **)error;
 @end

io/DecryptedInputStream.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2010, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2011, Stefan Reitshamer http://www.haystacksoftware.com
  
  All rights reserved.
  
@@ -30,12 +30,17 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */ 
 
+#import <Cocoa/Cocoa.h>
+
 #import "DecryptedInputStream.h"
 #include <openssl/evp.h>
 
 @implementation DecryptedInputStream
-- (id)initWithInputStream:(id <InputStream>)theIS cipherName:(NSString *)theCipherName key:(NSString *)theKey error:(NSError **)error {
+- (id)initWithInputStream:(id <InputStream>)theIS cryptoKey:(CryptoKey *)theCryptoKey error:(NSError **)error {
-    self = [super initWithCryptInitFunc:&EVP_DecryptInit cryptUpdateFunc:&EVP_DecryptUpdate cryptFinalFunc:&EVP_DecryptFinal inputStream:theIS cipherName:theCipherName key:theKey error:error];
+    self = [super initWithCryptInitFunc:&EVP_DecryptInit cryptUpdateFunc:&EVP_DecryptUpdate cryptFinalFunc:&EVP_DecryptFinal inputStream:theIS cryptoKey:theCryptoKey label:@"decrypt" error:error];
     return self;
 }
+- (NSString *)description {
+    return [NSString stringWithFormat:@"<DecryptedInputStream %@>", is];
+}
 @end