Updated to be compatible with backups created by Arq 5.

ArqRestoreCommand.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,14 +31,14 @@
  */
 
 
-#import "S3RestorerDelegate.h"
+
+#import "StandardRestorerDelegate.h"
 #import "S3GlacierRestorerDelegate.h"
 #import "GlacierRestorerDelegate.h"
 @class Target;
 
 
-@interface ArqRestoreCommand : NSObject <S3RestorerDelegate, S3GlacierRestorerDelegate, GlacierRestorerDelegate> {
+@interface ArqRestoreCommand : NSObject <StandardRestorerDelegate, S3GlacierRestorerDelegate, GlacierRestorerDelegate> {
-    Target *target;
     unsigned long long maxRequested;
     unsigned long long maxTransfer;
 }

ArqRestoreCommand.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #import "ArqRestoreCommand.h"
 #import "Target.h"
 #import "AWSRegion.h"
@@ -38,24 +40,27 @@
 #import "UserAndComputer.h"
 #import "Bucket.h"
 #import "Repo.h"
-#import "S3RestorerParamSet.h"
+#import "StandardRestorerParamSet.h"
 #import "Tree.h"
 #import "Commit.h"
+#import "Node.h"
 #import "BlobKey.h"
-#import "S3Restorer.h"
+#import "StandardRestorer.h"
 #import "S3GlacierRestorerParamSet.h"
 #import "S3GlacierRestorer.h"
 #import "GlacierRestorerParamSet.h"
 #import "GlacierRestorer.h"
 #import "S3AuthorizationProvider.h"
+#import "S3AuthorizationProviderFactory.h"
+#import "NSString_extra.h"
+#import "TargetFactory.h"
+#import "RegexKitLite.h"
+#import "BackupSet.h"
+#import "ExePath.h"
+#import "AWSRegion.h"
 
 
 @implementation ArqRestoreCommand
-- (void)dealloc {
-    [target release];
-    [super dealloc];
-}
-
 - (NSString *)errorDomain {
     return @"ArqRestoreCommandErrorDomain";
 }
@@ -71,60 +76,29 @@
         return NO;
     }
     
-    int index = 1;
+    if ([args count] > 3 && [[args objectAtIndex:1] isEqualToString:@"-l"]) {
-    if ([[args objectAtIndex:1] isEqualToString:@"-l"]) {
+        [[HSLog sharedHSLog] setHSLogLevel:[HSLog hsLogLevelForName:[args objectAtIndex:2]]];
-        if ([args count] < 4) {
+        args = [NSMutableArray arrayWithArray:[args subarrayWithRange:NSMakeRange(2, [args count] - 2)]];
-            SETNSERROR([self errorDomain], ERROR_USAGE, @"missing arguments");
-            return NO;
-        }
-        setHSLogLevel(hsLogLevelForName([args objectAtIndex:2]));
-        index += 2;
     }
     
-    NSString *cmd = [args objectAtIndex:index];
+    NSString *cmd = [args objectAtIndex:1];
     
-    int targetParamsIndex = index + 1;
+    if ([cmd isEqualToString:@"listtargets"]) {
-    if ([cmd isEqualToString:@"listcomputers"]) {
+        return [self listTargets:error];
-        // Valid command, but no additional args.
+    } else if ([cmd isEqualToString:@"addtarget"]) {
-        
+        return [self addTarget:args error:error];
+    } else if ([cmd isEqualToString:@"deletetarget"]) {
+        return [self deleteTarget:args error:error];
+    } else if ([cmd isEqualToString:@"listcomputers"]) {
+        return [self listComputers:args error:error];
     } else if ([cmd isEqualToString:@"listfolders"]) {
-        if ((argc - targetParamsIndex) < 2) {
+        return [self listFolders:args error:error];
-            SETNSERROR([self errorDomain], ERROR_USAGE, @"missing arguments for listfolders command");
+    } else if ([cmd isEqualToString:@"listtree"]) {
-            return NO;
+        return [self listTree:args error:error];
-        }
-        targetParamsIndex += 2;
     } else if ([cmd isEqualToString:@"restore"]) {
-        if ((argc - targetParamsIndex) < 4) {
+        return [self restore:args error:error];
-            SETNSERROR([self errorDomain], ERROR_USAGE, @"missing arguments");
+    } else if ([cmd isEqualToString:@"clearcache"]) {
-            return NO;
+        return [self clearCache:args error:error];
-        }
-        targetParamsIndex += 4;
-    } else {
-        SETNSERROR([self errorDomain], ERROR_USAGE, @"unknown command: %@", cmd);
-        return NO;
-    }
-    
-    if (targetParamsIndex >= argc) {
-        SETNSERROR([self errorDomain], ERROR_USAGE, @"missing target type params");
-        return NO;
-    }
-    target = [[self targetForParams:[args subarrayWithRange:NSMakeRange(targetParamsIndex, argc - targetParamsIndex)] error:error] retain];
-    if (target == nil) {
-        return NO;
-    }
-    
-    if ([cmd isEqualToString:@"listcomputers"]) {
-        if (![self listComputers:error]) {
-            return NO;
-        }
-    } else if ([cmd isEqualToString:@"listfolders"]) {
-        if (![self listBucketsForComputerUUID:[args objectAtIndex:index+1] encryptionPassword:[args objectAtIndex:index+2] error:error]) {
-            return NO;
-        }
-    } else if ([cmd isEqualToString:@"restore"]) {
-        if (![self restoreComputerUUID:[args objectAtIndex:index+1] bucketUUID:[args objectAtIndex:index+3] encryptionPassword:[args objectAtIndex:index+2] restoreBytesPerSecond:[args objectAtIndex:index+4] error:error]) {
-            return NO;
-        }
     } else {
         SETNSERROR([self errorDomain], ERROR_USAGE, @"unknown command: %@", cmd);
         return NO;
@@ -135,156 +109,126 @@
 
 
 #pragma mark internal
-- (Target *)targetForParams:(NSArray *)theParams error:(NSError **)error {
+- (BOOL)listTargets:(NSError **)error {
-    NSString *theTargetType = [theParams objectAtIndex:0];
+    printf("%-20s %s\n", "nickname:", "url:");
-    
+    for (Target *target in [[TargetFactory sharedTargetFactory] sortedTargets]) {
-    Target *ret = nil;
+        printf("%-20s %s\n", [[target nickname] UTF8String], [[[target endpoint] description] UTF8String]);
-    if ([theTargetType isEqualToString:@"aws"]) {
+    }
-        if ([theParams count] != 4) {
+    return YES;
-            SETNSERROR([self errorDomain], ERROR_USAGE, @"invalid aws parameters");
+}
-            return nil;
-        }
-        
-        NSString *theAccessKey = [theParams objectAtIndex:1];
-        NSString *theSecretKey = [theParams objectAtIndex:2];
-        NSString *theBucketName = [theParams objectAtIndex:3];
-        AWSRegion *awsRegion = [self awsRegionForAccessKey:theAccessKey secretKey:theSecretKey bucketName:theBucketName error:error];
-        if (awsRegion == nil) {
-            return nil;
-        }
-        NSURL *s3Endpoint = [awsRegion s3EndpointWithSSL:YES];
-        int port = [[s3Endpoint port] intValue];
-        NSString *portString = @"";
-        if (port != 0) {
-            portString = [NSString stringWithFormat:@":%d", port];
-        }
-        NSURL *targetEndpoint = [NSURL URLWithString:[NSString stringWithFormat:@"%@://%@@%@%@/%@", [s3Endpoint scheme], theAccessKey, [s3Endpoint host], portString, theBucketName]];
-        ret = [[[Target alloc] initWithEndpoint:targetEndpoint secret:theSecretKey passphrase:nil] autorelease];
-    } else if ([theTargetType isEqualToString:@"sftp"]) {
-        if ([theParams count] != 6 && [theParams count] != 7) {
-            SETNSERROR([self errorDomain], ERROR_USAGE, @"invalid sftp parameters");
-            return nil;
-        }
-        
-        NSString *hostname = [theParams objectAtIndex:1];
-        int port = [[theParams objectAtIndex:2] intValue];
-        NSString *path = [theParams objectAtIndex:3];
-        NSString *username = [theParams objectAtIndex:4];
-        NSString *secret = [theParams objectAtIndex:5];
-        NSString *keyfilePassphrase = [theParams count] > 6 ? [theParams objectAtIndex:6] : nil;
-        
-        if (![path hasPrefix:@"/"]) {
-            path = [@"/~/" stringByAppendingString:path];
-        }
-        NSString *escapedPath = (NSString *)CFURLCreateStringByAddingPercentEscapes(NULL, (CFStringRef)path, NULL, (CFStringRef)@"!*'();:@&=+$,?%#[]", kCFStringEncodingUTF8);
-        NSString *escapedUsername = (NSString *)CFURLCreateStringByAddingPercentEscapes(NULL, (CFStringRef)username, NULL, (CFStringRef)@"!*'();:@&=+$,?%#[]", kCFStringEncodingUTF8);
-        NSURL *endpoint = [NSURL URLWithString:[NSString stringWithFormat:@"sftp://%@@%@:%d%@", escapedUsername, hostname, port, escapedPath]];
 
-        ret = [[[Target alloc] initWithEndpoint:endpoint secret:secret passphrase:keyfilePassphrase] autorelease];
+- (BOOL)addTarget:(NSArray *)args error:(NSError **)error {
-    } else if ([theTargetType isEqualToString:@"greenqloud"]
+    if ([args count] < 5) {
-               || [theTargetType isEqualToString:@"dreamobjects"]
+        SETNSERROR([self errorDomain], ERROR_USAGE, @"missing arguments");
-               || [theTargetType isEqualToString:@"googlecloudstorage"]) {
+        return NO;
-        if ([theParams count] != 4) {
+    }
-            SETNSERROR([self errorDomain], ERROR_USAGE, @"invalid %@ parameters", theTargetType);
+    NSString *targetUUID = [NSString stringWithRandomUUID];
-            return nil;
+    NSString *targetNickname = [args objectAtIndex:2];
+    NSString *targetType = [args objectAtIndex:3];
+    
+    NSURL *endpoint = nil;
+    NSString *secret = nil;
+    NSString *passphrase = nil;
+    NSString *oAuth2ClientId = nil;
+    NSString *oAuth2ClientSecret = nil;
+    NSString *oAuth2RedirectURI = nil;
+    
+    if ([targetType isEqualToString:@"aws"]) {
+        if ([args count] != 6) {
+            SETNSERROR([self errorDomain], ERROR_USAGE, @"invalid arguments");
+            return NO;
         }
         
-        NSString *theAccessKey = [theParams objectAtIndex:1];
+        NSString *accessKeyId = [args objectAtIndex:4];
-        NSString *theSecretKey = [theParams objectAtIndex:2];
+        AWSRegion *usEast1 = [AWSRegion usEast1];
-        NSString *theBucketName = [theParams objectAtIndex:3];
+        NSString *urlString = [NSString stringWithFormat:@"https://%@@%@/any_bucket", accessKeyId, [[usEast1 s3EndpointWithSSL:NO] host]];
-        NSString *theHostname = nil;
+        
-        if ([theTargetType isEqualToString:@"greenqloud"]) {
+        endpoint = [NSURL URLWithString:urlString];
-            theHostname = @"s.greenqloud.com";
+        secret = [args objectAtIndex:5];
-        } else if ([theTargetType isEqualToString:@"dreamobjects"]) {
+        
-            theHostname = @"objects.dreamhost.com";
+    } else if ([targetType isEqualToString:@"local"]) {
-        } else if ([theTargetType isEqualToString:@"googlecloudstorage"]) {
+        if ([args count] != 5) {
-            theHostname = @"storage.googleapis.com";
+            SETNSERROR([self errorDomain], ERROR_USAGE, @"invalid arguments");
-        } else {
+            return NO;
-            SETNSERROR([self errorDomain], ERROR_USAGE, @"no hostname for target type: %@", theTargetType);
-            return nil;
         }
         
-        NSURL *endpoint = [NSURL URLWithString:[NSString stringWithFormat:@"https://%@@%@/%@", theAccessKey, theHostname, theBucketName]];
+        endpoint = [NSURL fileURLWithPath:[args objectAtIndex:4]];
-        ret = [[[Target alloc] initWithEndpoint:endpoint secret:theSecretKey passphrase:nil] autorelease];
+        secret = @"unused";
-    } else if ([theTargetType isEqualToString:@"s3compatible"]) {
-        if ([theParams count] != 5) {
-            SETNSERROR([self errorDomain], ERROR_USAGE, @"invalid %@ parameters", theTargetType);
-            return nil;
-        }
-        
-        NSURL *theURL = [NSURL URLWithString:[theParams objectAtIndex:1]];
-        if (theURL == nil) {
-            SETNSERROR([self errorDomain], ERROR_USAGE, @"invalid url %@", [theParams objectAtIndex:1]);
-            return nil;
-        }
-        
-        NSString *theAccessKey = [theParams objectAtIndex:2];
-        NSString *theSecretKey = [theParams objectAtIndex:3];
-        NSString *theBucketName = [theParams objectAtIndex:4];
-        
-        NSMutableString *urlString = [NSMutableString stringWithString:[theURL scheme]];
-        [urlString appendFormat:@"://%@@%@", theAccessKey, [theURL host]];
-        NSNumber *port = [theURL port];
-        if (port != nil) {
-            [urlString appendFormat:@":%d", [port intValue]];
-        }
-        [urlString appendString:@"/"];
-        [urlString appendString:theBucketName];
-        NSURL *endpoint = [NSURL URLWithString:urlString];
-        ret = [[[Target alloc] initWithEndpoint:endpoint secret:theSecretKey passphrase:nil] autorelease];
-        
-    } else if ([theTargetType isEqualToString:@"googledrive"]) {
-        if ([theParams count] != 3) {
-            SETNSERROR([self errorDomain], ERROR_USAGE, @"invalid googledrive parameters");
-            return nil;
-        }
-        
-        NSString *theRefreshToken = [theParams objectAtIndex:1];
-        NSString *thePath = [theParams objectAtIndex:2];
-        
-        NSString *escapedPath = (NSString *)CFURLCreateStringByAddingPercentEscapes(kCFAllocatorDefault, (CFStringRef)thePath, CFSTR("/"), CFSTR("@?=&+"), kCFStringEncodingUTF8);
-        [escapedPath autorelease];
-        
-        NSURL *endpoint = [NSURL URLWithString:[NSString stringWithFormat:@"googledrive://unknown_email_address@www.googleapis.com%@", escapedPath]];
-        ret = [[[Target alloc] initWithEndpoint:endpoint secret:theRefreshToken passphrase:nil] autorelease];
     } else {
-        SETNSERROR([self errorDomain], ERROR_USAGE, @"unknown target type: %@", theTargetType);
+        SETNSERROR([self errorDomain], -1, @"unknown target type: %@", targetType);
-        return nil;
+        return NO;
     }
-    return ret;
-}
-
-- (AWSRegion *)awsRegionForAccessKey:(NSString *)theAccessKey secretKey:(NSString *)theSecretKey bucketName:(NSString *)theBucketName error:(NSError **)error {
-    S3AuthorizationProvider *sap = [[[S3AuthorizationProvider alloc] initWithAccessKey:theAccessKey secretKey:theSecretKey] autorelease];
-    NSURL *endpoint = [[AWSRegion usEast1] s3EndpointWithSSL:YES];
-    S3Service *s3 = [[[S3Service alloc] initWithS3AuthorizationProvider:sap endpoint:endpoint useAmazonRRS:NO] autorelease];
     
-    NSString *location = [s3 locationOfS3Bucket:theBucketName targetConnectionDelegate:nil error:error];
+    Target *target = [[[Target alloc] initWithUUID:targetUUID nickname:targetNickname endpoint:endpoint awsRequestSignatureVersion:4] autorelease];
-    if (location == nil) {
+    [target setOAuth2ClientId:oAuth2ClientId];
-        return nil;
+    [target setOAuth2RedirectURI:oAuth2RedirectURI];
+    if (![[TargetFactory sharedTargetFactory] saveTarget:target error:error]) {
+        return NO;
     }
-    return [AWSRegion regionWithLocation:location];
+    if (![target setSecret:secret trustedAppPaths:[NSArray arrayWithObject:[ExePath exePath]] error:error]) {
+        return NO;
+    }
+    if (passphrase != nil) {
+        if (![target setPassphrase:passphrase trustedAppPaths:[NSArray arrayWithObject:[ExePath exePath]] error:error]) {
+            return NO;
+        }
+    }
+    if (oAuth2ClientSecret != nil) {
+        if (![target setOAuth2ClientSecret:oAuth2ClientSecret trustedAppPaths:[NSArray arrayWithObject:[ExePath exePath]] error:error]) {
+            return NO;
+        }
+    }
+    
+    return YES;
+}
+- (BOOL)deleteTarget:(NSArray *)args error:(NSError **)error {
+    if ([args count] != 3) {
+        SETNSERROR([self errorDomain], ERROR_USAGE, @"invalid arguments");
+        return NO;
+    }
+    Target *target = [[TargetFactory sharedTargetFactory] targetWithNickname:[args objectAtIndex:2]];
+    if (target == nil) {
+        SETNSERROR([self errorDomain], ERROR_NOT_FOUND, @"target not found");
+        return NO;
+    }
+    TargetConnection *conn = [[target newConnection:error] autorelease];
+    if (conn == nil) {
+        return NO;
+    }
+    if (![conn clearAllCachedData:error]) {
+        return NO;
+    }
+
+    return [[TargetFactory sharedTargetFactory] deleteTarget:target error:error];
 }
 
-- (BOOL)listComputers:(NSError **)error {
+- (BOOL)listComputers:(NSArray *)args error:(NSError **)error {
-    NSArray *expandedTargetList = [self expandedTargetList:error];
+    if ([args count] != 3) {
+        SETNSERROR([self errorDomain], ERROR_USAGE, @"invalid arguments");
+        return NO;
+    }
+    Target *target = [[TargetFactory sharedTargetFactory] targetWithNickname:[args objectAtIndex:2]];
+    if (target == nil) {
+        SETNSERROR([self errorDomain], ERROR_NOT_FOUND, @"target not found");
+        return NO;
+    }
+    NSArray *expandedTargetList = [self expandedTargetListForTarget:target error:error];
     if (expandedTargetList == nil) {
         return NO;
     }
     
-    NSMutableArray *ret = [NSMutableArray array];
     for (Target *theTarget in expandedTargetList) {
         NSError *myError = nil;
         HSLogDebug(@"getting backup sets for %@", theTarget);
         
-        NSArray *backupSets = [BackupSet allBackupSetsForTarget:theTarget targetConnectionDelegate:nil error:&myError];
+        NSArray *backupSets = [BackupSet allBackupSetsForTarget:theTarget targetConnectionDelegate:nil activityListener:nil error:&myError];
         if (backupSets == nil) {
             if ([myError isErrorWithDomain:[S3Service errorDomain] code:S3SERVICE_ERROR_AMAZON_ERROR] && [[[myError userInfo] objectForKey:@"HTTPStatusCode"] intValue] == 403) {
                 HSLogError(@"access denied getting backup sets for %@", theTarget);
             } else {
                 HSLogError(@"error getting backup sets for %@: %@", theTarget, myError);
                 SETERRORFROMMYERROR;
-                return nil;
+                return NO;
             }
         } else {
             printf("target: %s\n", [[theTarget endpointDisplayName] UTF8String]);
@@ -294,65 +238,32 @@
             }
         }
     }
-    return ret;
+    return YES;
-}
-- (NSArray *)expandedTargetList:(NSError **)error {
-    NSMutableArray *expandedTargetList = [NSMutableArray arrayWithObject:target];
-//    if ([target targetType] == kTargetAWS
-//        || [target targetType] == kTargetDreamObjects
-//        || [target targetType] == kTargetGoogleCloudStorage
-//        || [target targetType] == kTargetGreenQloud
-//        || [target targetType] == kTargetS3Compatible) {
-//        NSError *myError = nil;
-//        NSArray *targets = [self expandedTargetsForS3Target:target error:&myError];
-//        if (targets == nil) {
-//            HSLogError(@"failed to expand target list for %@: %@", target, myError);
-//        } else {
-//            [expandedTargetList setArray:targets];
-//            HSLogDebug(@"expandedTargetList is now: %@", expandedTargetList);
-//        }
-//    }
-    return expandedTargetList;
-}
-- (NSArray *)expandedTargetsForS3Target:(Target *)theTarget error:(NSError **)error {
-    S3Service *s3 = [theTarget s3:error];
-    if (s3 == nil) {
-        return nil;
-    }
-    NSArray *s3BucketNames = [s3 s3BucketNamesWithTargetConnectionDelegate:nil error:error];
-    if (s3BucketNames == nil) {
-        return nil;
-    }
-    HSLogDebug(@"s3BucketNames for %@: %@", theTarget, s3BucketNames);
-    
-    NSURL *originalEndpoint = [theTarget endpoint];
-    NSMutableArray *ret = [NSMutableArray array];
-    
-    for (NSString *s3BucketName in s3BucketNames) {
-        NSURL *endpoint = nil;
-        if ([theTarget targetType] == kTargetAWS) {
-            NSString *location = [s3 locationOfS3Bucket:s3BucketName targetConnectionDelegate:nil error:error];
-            if (location == nil) {
-                return nil;
-            }
-            AWSRegion *awsRegion = [AWSRegion regionWithLocation:location];
-            HSLogDebug(@"awsRegion for s3BucketName %@: %@", s3BucketName, location);
-            
-            NSURL *s3Endpoint = [awsRegion s3EndpointWithSSL:YES];
-            HSLogDebug(@"s3Endpoint: %@", s3Endpoint);
-            endpoint = [NSURL URLWithString:[NSString stringWithFormat:@"https://%@@%@/%@", [originalEndpoint user], [s3Endpoint host], s3BucketName]];
-        } else {
-            endpoint = [NSURL URLWithString:[NSString stringWithFormat:@"%@://%@@%@/%@", [originalEndpoint scheme], [originalEndpoint user], [originalEndpoint host], s3BucketName]];
-        }
-        HSLogDebug(@"endpoint: %@", endpoint);
-        
-        Target *theTarget = [[[Target alloc] initWithEndpoint:endpoint secret:[theTarget secret:NULL] passphrase:[theTarget passphrase:NULL]] autorelease];
-        [ret addObject:theTarget];
-    }
-    return ret;
 }
 
-- (BOOL)listBucketsForComputerUUID:(NSString *)theComputerUUID encryptionPassword:(NSString *)theEncryptionPassword error:(NSError **)error {
+
+- (BOOL)listFolders:(NSArray *)args error:(NSError **)error {
+    if ([args count] != 5) {
+        SETNSERROR([self errorDomain], ERROR_USAGE, @"invalid arguments");
+        return NO;
+    }
+    Target *target = [[TargetFactory sharedTargetFactory] targetWithNickname:[args objectAtIndex:2]];
+    if (target == nil) {
+        SETNSERROR([self errorDomain], ERROR_NOT_FOUND, @"target not found");
+        return NO;
+    }
+    
+    NSString *theComputerUUID = [args objectAtIndex:3];
+    NSString *theEncryptionPassword = [args objectAtIndex:4];
+    
+    BackupSet *backupSet = [self backupSetForTarget:target computerUUID:theComputerUUID error:error];
+    if (backupSet == nil) {
+        return NO;
+    }
+    
+    // Reset Target:
+    target = [backupSet target];
+    
     NSArray *buckets = [Bucket bucketsWithTarget:target computerUUID:theComputerUUID encryptionPassword:theEncryptionPassword targetConnectionDelegate:nil error:error];
     if (buckets == nil) {
         return NO;
@@ -366,151 +277,359 @@
         printf("\t\tuuid %s\n", [[bucket bucketUUID] UTF8String]);
         
     }
-    
     return YES;
 }
-- (BOOL)restoreComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID encryptionPassword:(NSString *)theEncryptionPassword restoreBytesPerSecond:(NSString *)theRestoreBytesPerSecond error:(NSError **)error {
+- (BOOL)listTree:(NSArray *)args error:(NSError **)error {
-    Bucket *myBucket = nil;
+    if ([args count] != 6) {
-    NSArray *expandedTargetList = [self expandedTargetList:error];
+        SETNSERROR([self errorDomain], ERROR_USAGE, @"invalid arguments");
-    if (expandedTargetList == nil) {
         return NO;
     }
-    for (Target *theTarget in expandedTargetList) {
+    Target *target = [[TargetFactory sharedTargetFactory] targetWithNickname:[args objectAtIndex:2]];
-        NSArray *buckets = [Bucket bucketsWithTarget:theTarget computerUUID:theComputerUUID encryptionPassword:theEncryptionPassword targetConnectionDelegate:nil error:error];
+    if (target == nil) {
-        if (buckets == nil) {
+        SETNSERROR([self errorDomain], ERROR_NOT_FOUND, @"target not found");
-            return NO;
+        return NO;
-        }
+    }
-        for (Bucket *bucket in buckets) {
+    
-            if ([[bucket bucketUUID] isEqualToString:theBucketUUID]) {
+    NSString *theComputerUUID = [args objectAtIndex:3];
-                myBucket = bucket;
+    NSString *theEncryptionPassword = [args objectAtIndex:4];
-                break;
+    NSString *theBucketUUID = [args objectAtIndex:5];
-            }
+    
-        }
+    BackupSet *backupSet = [self backupSetForTarget:target computerUUID:theComputerUUID error:error];
-        
+    if (backupSet == nil) {
-        if (myBucket != nil) {
+        return NO;
+    }
+    
+    // Reset Target:
+    target = [backupSet target];
+    
+    NSArray *buckets = [Bucket bucketsWithTarget:target computerUUID:theComputerUUID encryptionPassword:theEncryptionPassword targetConnectionDelegate:nil error:error];
+    if (buckets == nil) {
+        return NO;
+    }
+    Bucket *matchingBucket = nil;
+    for (Bucket *bucket in buckets) {
+        if ([[bucket bucketUUID] isEqualToString:theBucketUUID]) {
+            matchingBucket = bucket;
             break;
         }
     }
-    if (myBucket == nil) {
+    if (matchingBucket == nil) {
         SETNSERROR([self errorDomain], ERROR_NOT_FOUND, @"folder %@ not found", theBucketUUID);
         return NO;
     }
     
-    Repo *repo = [[[Repo alloc] initWithBucket:myBucket encryptionPassword:theEncryptionPassword targetUID:getuid() targetGID:getgid() loadExistingMutablePackFiles:NO targetConnectionDelegate:nil repoDelegate:nil error:error] autorelease];
+    printf("target   %s\n", [[target endpointDisplayName] UTF8String]);
+    printf("computer %s\n", [theComputerUUID UTF8String]);
+    printf("folder   %s\n", [theBucketUUID UTF8String]);
+    
+    Repo *repo = [[[Repo alloc] initWithBucket:matchingBucket encryptionPassword:theEncryptionPassword targetConnectionDelegate:nil repoDelegate:nil activityListener:nil error:error] autorelease];
     if (repo == nil) {
         return NO;
     }
+    BlobKey *headBlobKey = [repo headBlobKey:error];
+    if (headBlobKey == nil) {
+        return NO;
+    }
+    Commit *head = [repo commitForBlobKey:headBlobKey error:error];
+    if (head == nil) {
+        return NO;
+    }
+    Tree *rootTree = [repo treeForBlobKey:[head treeBlobKey] error:error];
+    if (rootTree == nil) {
+        return NO;
+    }
+    return [self printTree:rootTree repo:repo relativePath:@"" error:error];
+}
+- (BOOL)printTree:(Tree *)theTree repo:(Repo *)theRepo relativePath:(NSString *)theRelativePath error:(NSError **)error {
+    for (NSString *childName in [theTree childNodeNames]) {
+        NSString *childRelativePath = [theRelativePath stringByAppendingFormat:@"/%@", childName];
+        Node *childNode = [theTree childNodeWithName:childName];
+        if ([childNode isTree]) {
+            printf("%s:\n", [childRelativePath UTF8String]);
+            Tree *childTree = [theRepo treeForBlobKey:[childNode treeBlobKey] error:error];
+            if (childTree == nil) {
+                return NO;
+            }
+            if (![self printTree:childTree
+                            repo:theRepo
+                    relativePath:childRelativePath
+                           error:error]) {
+                return NO;
+            }
+        } else {
+            printf("%s\n", [childRelativePath UTF8String]);
+        }
+    }
+    return YES;
+}
+
+- (BOOL)restore:(NSArray *)args error:(NSError **)error {
+    if ([args count] != 6) {
+        SETNSERROR([self errorDomain], ERROR_USAGE, @"invalid arguments");
+        return NO;
+    }
+    Target *target = [[TargetFactory sharedTargetFactory] targetWithNickname:[args objectAtIndex:2]];
+    if (target == nil) {
+        SETNSERROR([self errorDomain], ERROR_NOT_FOUND, @"target not found");
+        return NO;
+    }
     
+    NSString *theComputerUUID = [args objectAtIndex:3];
+    NSString *theEncryptionPassword = [args objectAtIndex:4];
+    NSString *theBucketUUID = [args objectAtIndex:5];
     
+    BackupSet *backupSet = [self backupSetForTarget:target computerUUID:theComputerUUID error:error];
+    if (backupSet == nil) {
+        return NO;
+    }
+    
+    // Reset Target:
+    target = [backupSet target];
+    
+    NSArray *buckets = [Bucket bucketsWithTarget:target computerUUID:theComputerUUID encryptionPassword:theEncryptionPassword targetConnectionDelegate:nil error:error];
+    if (buckets == nil) {
+        return NO;
+    }
+    Bucket *matchingBucket = nil;
+    for (Bucket *bucket in buckets) {
+        if ([[bucket bucketUUID] isEqualToString:theBucketUUID]) {
+            matchingBucket = bucket;
+            break;
+        }
+    }
+    if (matchingBucket == nil) {
+        SETNSERROR([self errorDomain], ERROR_NOT_FOUND, @"folder %@ not found", theBucketUUID);
+        return NO;
+    }
+    
+    printf("target   %s\n", [[target endpointDisplayName] UTF8String]);
+    printf("computer %s\n", [theComputerUUID UTF8String]);
+    printf("folder   %s\n", [theBucketUUID UTF8String]);
+    
+    Repo *repo = [[[Repo alloc] initWithBucket:matchingBucket encryptionPassword:theEncryptionPassword targetConnectionDelegate:nil repoDelegate:nil activityListener:nil error:error] autorelease];
+    if (repo == nil) {
+        return NO;
+    }
     BlobKey *commitBlobKey = [repo headBlobKey:error];
     if (commitBlobKey == nil) {
         return NO;
     }
-    Commit *commit = [repo commitForBlobKey:commitBlobKey dataSize:NULL error:error];
+    Commit *commit = [repo commitForBlobKey:commitBlobKey error:error];
     if (commit == nil) {
         return NO;
     }
     
-    NSString *destinationPath = [[[NSFileManager defaultManager] currentDirectoryPath] stringByAppendingPathComponent:[[myBucket localPath] lastPathComponent]];
+    NSString *destinationPath = [[[NSFileManager defaultManager] currentDirectoryPath] stringByAppendingPathComponent:[[matchingBucket localPath] lastPathComponent]];
     if ([[NSFileManager defaultManager] fileExistsAtPath:destinationPath]) {
         SETNSERROR([self errorDomain], -1, @"%@ already exists", destinationPath);
         return NO;
     }
     
     
-    printf("target   %s\n", [[[myBucket target] endpointDisplayName] UTF8String]);
+    printf("\nrestoring folder %s to %s\n\n", [[matchingBucket localPath] UTF8String], [destinationPath UTF8String]);
-    printf("computer %s\n", [[myBucket computerUUID] UTF8String]);
+    
-    printf("\nrestoring folder %s to %s\n\n", [[myBucket localPath] UTF8String], [destinationPath UTF8String]);
+    int bytesPerSecond = 500000;
     
     AWSRegion *region = [AWSRegion regionWithS3Endpoint:[target endpoint]];
     BOOL isGlacierDestination = [region supportsGlacier];
-    if ([myBucket storageType] == StorageTypeGlacier && isGlacierDestination) {
+    if ([matchingBucket storageType] == StorageTypeGlacier && isGlacierDestination) {
-        int bytesPerSecond = [theRestoreBytesPerSecond intValue];
+        GlacierRestorerParamSet *paramSet = [[[GlacierRestorerParamSet alloc] initWithBucket:matchingBucket
-        if (bytesPerSecond == 0) {
-            SETNSERROR([self errorDomain], -1, @"invalid bytes_per_second %@", theRestoreBytesPerSecond);
-            return NO;
-        }
-        
-        GlacierRestorerParamSet *paramSet = [[[GlacierRestorerParamSet alloc] initWithBucket:myBucket
                                                                           encryptionPassword:theEncryptionPassword
                                                                       downloadBytesPerSecond:bytesPerSecond
+                                                                        glacierRetrievalTier:GLACIER_RETRIEVAL_TIER_EXPEDITED
                                                                                commitBlobKey:commitBlobKey
-                                                                                rootItemName:[[myBucket localPath] lastPathComponent]
+                                                                                rootItemName:[[matchingBucket localPath] lastPathComponent]
                                                                                  treeVersion:CURRENT_TREE_VERSION
-                                                                            treeIsCompressed:[[commit treeBlobKey] compressed]
                                                                                  treeBlobKey:[commit treeBlobKey]
                                                                                     nodeName:nil targetUID:getuid()
                                                                                    targetGID:getgid()
                                                                           useTargetUIDAndGID:YES
                                                                              destinationPath:destinationPath
-                                                                                    logLevel:global_hslog_level] autorelease];
+                                                                                    logLevel:[[HSLog sharedHSLog] hsLogLevel]] autorelease];
         [[[GlacierRestorer alloc] initWithGlacierRestorerParamSet:paramSet delegate:self] autorelease];
         
-    } else if ([myBucket storageType] == StorageTypeS3Glacier && isGlacierDestination) {
+    } else if ([matchingBucket storageType] == StorageTypeS3Glacier && isGlacierDestination) {
-        int bytesPerSecond = [theRestoreBytesPerSecond intValue];
+        S3GlacierRestorerParamSet *paramSet = [[[S3GlacierRestorerParamSet alloc] initWithBucket:matchingBucket
-        if (bytesPerSecond == 0) {
-            SETNSERROR([self errorDomain], -1, @"invalid bytes_per_second %@", theRestoreBytesPerSecond);
-            return NO;
-        }
-        
-        S3GlacierRestorerParamSet *paramSet = [[[S3GlacierRestorerParamSet alloc] initWithBucket:myBucket
                                                                               encryptionPassword:theEncryptionPassword
                                                                           downloadBytesPerSecond:bytesPerSecond
+                                                                            glacierRetrievalTier:GLACIER_RETRIEVAL_TIER_EXPEDITED
                                                                                    commitBlobKey:commitBlobKey
-                                                                                    rootItemName:[[myBucket localPath] lastPathComponent]
+                                                                                    rootItemName:[[matchingBucket localPath] lastPathComponent]
                                                                                      treeVersion:CURRENT_TREE_VERSION
-                                                                                treeIsCompressed:[[commit treeBlobKey] compressed]
                                                                                      treeBlobKey:[commit treeBlobKey]
                                                                                         nodeName:nil
                                                                                        targetUID:getuid()
                                                                                        targetGID:getgid()
                                                                               useTargetUIDAndGID:YES
                                                                                  destinationPath:destinationPath
-                                                                                        logLevel:global_hslog_level] autorelease];
+                                                                                        logLevel:[[HSLog sharedHSLog] hsLogLevel]] autorelease];
         S3GlacierRestorer *restorer = [[[S3GlacierRestorer alloc] initWithS3GlacierRestorerParamSet:paramSet delegate:self] autorelease];
         [restorer run];
     } else {
-        S3RestorerParamSet *paramSet = [[[S3RestorerParamSet alloc] initWithBucket:myBucket
+        StandardRestorerParamSet *paramSet = [[[StandardRestorerParamSet alloc] initWithBucket:matchingBucket
-                                                                encryptionPassword:theEncryptionPassword
+                                                                            encryptionPassword:theEncryptionPassword
-                                                                     commitBlobKey:commitBlobKey
+                                                                                 commitBlobKey:commitBlobKey
-                                                                      rootItemName:[[myBucket localPath] lastPathComponent]
+                                                                                  rootItemName:[[matchingBucket localPath] lastPathComponent]
-                                                                       treeVersion:CURRENT_TREE_VERSION
+                                                                                   treeVersion:CURRENT_TREE_VERSION
-                                                                  treeIsCompressed:[[commit treeBlobKey] compressed]
+                                                                                   treeBlobKey:[commit treeBlobKey]
-                                                                       treeBlobKey:[commit treeBlobKey]
+                                                                                      nodeName:nil
-                                                                          nodeName:nil
+                                                                                     targetUID:getuid()
-                                                                         targetUID:getuid()
+                                                                                     targetGID:getgid()
-                                                                         targetGID:getgid()
+                                                                            useTargetUIDAndGID:YES
-                                                                useTargetUIDAndGID:YES
+                                                                               destinationPath:destinationPath
-                                                                   destinationPath:destinationPath
+                                                                                      logLevel:[[HSLog sharedHSLog] hsLogLevel]] autorelease];
-                                                                          logLevel:global_hslog_level] autorelease];
+        [[[StandardRestorer alloc] initWithParamSet:paramSet delegate:self] autorelease];
-        [[[S3Restorer alloc] initWithParamSet:paramSet delegate:self] autorelease];
     }
     
     return YES;
 }
+- (BOOL)clearCache:(NSArray *)args error:(NSError **)error {
+    if ([args count] != 3) {
+        SETNSERROR([self errorDomain], ERROR_USAGE, @"invalid arguments");
+        return NO;
+    }
+    Target *target = [[TargetFactory sharedTargetFactory] targetWithNickname:[args objectAtIndex:2]];
+    if (target == nil) {
+        SETNSERROR([self errorDomain], ERROR_NOT_FOUND, @"target not found");
+        return NO;
+    }
+    TargetConnection *conn = [[target newConnection:error] autorelease];
+    if (conn == nil) {
+        return NO;
+    }
+    return [conn clearAllCachedData:error];
+}
 
 
-#pragma mark S3RestorerDelegate
+
+- (BackupSet *)backupSetForTarget:(Target *)theInitialTarget computerUUID:(NSString *)theComputerUUID error:(NSError **)error {
+    NSArray *expandedTargetList = [self expandedTargetListForTarget:theInitialTarget error:error];
+    if (expandedTargetList == nil) {
+        return NO;
+    }
+    
+    for (Target *theTarget in expandedTargetList) {
+        NSError *myError = nil;
+        NSArray *backupSets = [BackupSet allBackupSetsForTarget:theTarget targetConnectionDelegate:nil activityListener:nil error:&myError];
+        if (backupSets == nil) {
+            if ([myError isErrorWithDomain:[S3Service errorDomain] code:S3SERVICE_ERROR_AMAZON_ERROR] && [[[myError userInfo] objectForKey:@"HTTPStatusCode"] intValue] == 403) {
+                HSLogError(@"access denied getting backup sets for %@", theTarget);
+            } else {
+                HSLogError(@"error getting backup sets for %@: %@", theTarget, myError);
+                SETERRORFROMMYERROR;
+                return NO;
+            }
+        } else {
+            for (BackupSet *backupSet in backupSets) {
+                if ([[backupSet computerUUID] isEqualToString:theComputerUUID]) {
+                    return backupSet;
+                }
+            }
+        }
+    }
+    SETNSERROR([self errorDomain], ERROR_NOT_FOUND, @"backup set %@ not found at target\n", theComputerUUID);
+    return nil;
+}
+
+- (NSArray *)expandedTargetListForTarget:(Target *)theTarget error:(NSError **)error {
+    NSArray *targets = nil;
+    
+    if ([theTarget targetType] == kTargetAWS) {
+        targets = [self expandedTargetsForS3Target:theTarget error:error];
+    } else {
+        targets = [NSArray arrayWithObject:theTarget];
+    }
+    return targets;
+}
+- (NSArray *)expandedTargetsForS3Target:(Target *)theTarget error:(NSError **)error {
+    NSString *theSecretKey = [theTarget secret:error];
+    if (theSecretKey == nil) {
+        return nil;
+    }
+    S3Service *s3 = nil;
+    if ([AWSRegion regionWithS3Endpoint:[theTarget endpoint]] != nil) {
+        // It's S3. Get bucket name list from us-east-1 region.
+        NSURL *usEast1Endpoint = [[AWSRegion usEast1] s3EndpointWithSSL:YES];
+        id <S3AuthorizationProvider> sap = [[S3AuthorizationProviderFactory sharedS3AuthorizationProviderFactory] providerForEndpoint:usEast1Endpoint
+                                                                                                                            accessKey:[[theTarget endpoint] user]
+                                                                                                                            secretKey:theSecretKey
+                                                                                                                     signatureVersion:4
+                                                                                                                            awsRegion:[AWSRegion usEast1]];
+        s3 = [[[S3Service alloc] initWithS3AuthorizationProvider:sap endpoint:usEast1Endpoint] autorelease];
+    } else {
+        s3 = [theTarget s3:error];
+        if (s3 == nil) {
+            return nil;
+        }
+    }
+    NSArray *s3BucketNames = [s3 s3BucketNamesWithTargetConnectionDelegate:nil error:error];
+    if (s3BucketNames == nil) {
+        return nil;
+    }
+    HSLogDebug(@"s3BucketNames for %@: %@", theTarget, s3BucketNames);
+    
+    NSURL *originalEndpoint = [theTarget endpoint];
+    NSMutableArray *ret = [NSMutableArray array];
+    
+    // WARNING: This is a hack! We're creating this Target using the same UUID so that the keychain lookups work!
+    NSString *targetUUID = [theTarget targetUUID];
+    
+    for (NSString *s3BucketName in s3BucketNames) {
+        NSURL *endpoint = nil;
+        if ([theTarget targetType] == kTargetAWS) {
+            NSError *myError = nil;
+            NSString *location = [s3 locationOfS3Bucket:s3BucketName targetConnectionDelegate:nil error:&myError];
+            if (location == nil) {
+                HSLogError(@"failed to get location of %@: %@", s3BucketName, myError);
+            } else {
+                AWSRegion *awsRegion = [AWSRegion regionWithLocation:location];
+                HSLogDebug(@"awsRegion for s3BucketName %@: %@", s3BucketName, location);
+                
+                NSURL *s3Endpoint = [awsRegion s3EndpointWithSSL:YES];
+                HSLogDebug(@"s3Endpoint: %@", s3Endpoint);
+                endpoint = [NSURL URLWithString:[NSString stringWithFormat:@"https://%@@%@/%@", [originalEndpoint user], [s3Endpoint host], s3BucketName]];
+            }
+        } else {
+            NSNumber *originalPort = [originalEndpoint port];
+            NSString *portString = (originalPort == nil) ? @"" : [NSString stringWithFormat:@":%d", [originalPort intValue]];
+            endpoint = [NSURL URLWithString:[NSString stringWithFormat:@"%@://%@@%@%@/%@", [originalEndpoint scheme], [originalEndpoint user], [originalEndpoint host], portString, s3BucketName]];
+        }
+        
+        if (endpoint != nil) {
+            HSLogDebug(@"endpoint: %@", endpoint);
+            
+            Target *target = [[[Target alloc] initWithUUID:targetUUID
+                                                  nickname:s3BucketName
+                                                  endpoint:endpoint
+                                awsRequestSignatureVersion:[theTarget awsRequestSignatureVersion]] autorelease];
+            [ret addObject:target];
+        }
+    }
+    return ret;
+}
+
+
+#pragma mark StandardRestorerDelegate
 // Methods return YES if cancel is requested.
 
-- (BOOL)s3RestorerMessageDidChange:(NSString *)message {
+- (BOOL)standardRestorerMessageDidChange:(NSString *)message {
     printf("status: %s\n", [message UTF8String]);
     return NO;
 }
-- (BOOL)s3RestorerBytesTransferredDidChange:(NSNumber *)theTransferred {
+- (BOOL)standardRestorerFileBytesRestoredDidChange:(NSNumber *)theTransferred {
     return NO;
 }
-- (BOOL)s3RestorerTotalBytesToTransferDidChange:(NSNumber *)theTotal {
+- (BOOL)standardRestorerTotalFileBytesToRestoreDidChange:(NSNumber *)theTotal {
     return NO;
 }
-- (BOOL)s3RestorerErrorMessage:(NSString *)theErrorMessage didOccurForPath:(NSString *)thePath {
+- (BOOL)standardRestorerErrorMessage:(NSString *)theErrorMessage didOccurForPath:(NSString *)thePath {
     printf("%s error: %s\n", [thePath UTF8String], [theErrorMessage UTF8String]);
     return NO;
 }
-- (BOOL)s3RestorerDidSucceed {
+- (BOOL)standardRestorerDidSucceed {
     return NO;
 }
-- (BOOL)s3RestorerDidFail:(NSError *)error {
+- (BOOL)standardRestorerDidFail:(NSError *)error {
     printf("failed: %s\n", [[error localizedDescription] UTF8String]);
     return NO;
 }

ArqSalt.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,6 +31,7 @@
  */
 
 
+
 #import "TargetConnection.h"
 @class AWSRegion;
 @class Target;
@@ -38,15 +39,12 @@
 
 @interface ArqSalt : NSObject {
     Target *target;
-    uid_t uid;
-    gid_t gid;
     NSString *computerUUID;
 }
 - (id)initWithTarget:(Target *)theTarget
-           targetUID:(uid_t)theTargetUID
+        computerUUID:(NSString *)theComputerUUID
-           targetGID:(gid_t)theTargetGID
+               error:(NSError **)error;
-        computerUUID:(NSString *)theComputerUUID;
+
-- (NSData *)saltWithTargetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
+- (BOOL)ensureSaltExistsAtTargetWithTargetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (BOOL)saveSalt:(NSData *)theSalt targetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
+- (NSData *)saltDataWithTargetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (NSData *)createSaltWithTargetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
 @end

ArqSalt.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,12 +31,14 @@
  */
 
 
+
 #import "ArqSalt.h"
 #import "NSFileManager_extra.h"
 #import "UserLibrary_Arq.h"
 #import "Target.h"
 #import "TargetConnection.h"
 #import "Streams.h"
+#import "CacheOwnership.h"
 
 
 #define SALT_LENGTH (8)
@@ -44,13 +46,15 @@
 
 @implementation ArqSalt
 - (id)initWithTarget:(Target *)theTarget
-           targetUID:(uid_t)theTargetUID
+        computerUUID:(NSString *)theComputerUUID
-           targetGID:(gid_t)theTargetGID
+               error:(NSError **)error {
-        computerUUID:(NSString *)theComputerUUID {
+    if (theComputerUUID == nil) {
+        SETNSERROR([self errorDomain], ERROR_NOT_FOUND, @"no computer UUID given for salt file");
+        return nil;
+    }
+    
     if (self = [super init]) {
         target = [theTarget retain];
-        uid = theTargetUID;
-        gid = theTargetGID;
         computerUUID = [theComputerUUID retain];
     }
     return self;
@@ -61,53 +65,82 @@
     [super dealloc];
 }
 
-- (NSData *)saltWithTargetConnectionDelegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+- (NSString *)errorDomain {
-    NSData *ret = [NSData dataWithContentsOfFile:[self localPath] options:NSUncachedRead error:error];
+    return @"ArqSaltErrorDomain";
-    if (ret == nil) {
-        id <TargetConnection> targetConnection = [target newConnection];
-        do {
-            ret = [targetConnection saltDataForComputerUUID:computerUUID delegate:theDelegate error:error];
-            if (ret != nil) {
-                NSError *myError = nil;
-                if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:[self localPath] targetUID:uid targetGID:gid error:&myError]
-                    || ![Streams writeData:ret atomicallyToFile:[self localPath] targetUID:uid targetGID:gid bytesWritten:NULL error:&myError]) {
-                    HSLogError(@"error caching salt data to %@: %@", [self localPath], myError);
-                }
-            }
-        } while(0);
-        [targetConnection release];
-    }
-    return ret;
 }
-- (BOOL)saveSalt:(NSData *)theSalt targetConnectionDelegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+
-    id <TargetConnection> targetConnection = [target newConnection];
+- (BOOL)ensureSaltExistsAtTargetWithTargetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    BOOL ret = YES;
+    TargetConnection *targetConnection = [target newConnection:error];
-    do {
+    if (targetConnection == nil) {
-        ret = [targetConnection setSaltData:theSalt forComputerUUID:computerUUID delegate:theDelegate error:error]
+        return NO;
-        && [[NSFileManager defaultManager] ensureParentPathExistsForPath:[self localPath] targetUID:uid targetGID:gid error:error]
+    }
-        && [Streams writeData:theSalt atomicallyToFile:[self localPath] targetUID:uid targetGID:gid bytesWritten:NULL error:error];
+    BOOL ret = [self ensureSaltExistsAtTargetWithTargetConnection:targetConnection targetConnectionDelegate:theDelegate error:error];
-    } while (0);
     [targetConnection release];
     return ret;
 }
-- (NSData *)createSaltWithTargetConnectionDelegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+- (BOOL)ensureSaltExistsAtTargetWithTargetConnection:(TargetConnection *)targetConnection targetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    NSData *theSalt = [self createRandomSalt];
+    NSError *myError = nil;
-    if (![self saveSalt:theSalt targetConnectionDelegate:theDelegate error:error]) {
+    NSData *saltData = [targetConnection saltDataForComputerUUID:computerUUID delegate:theDelegate error:&myError];
-        return nil;
+    if (saltData == nil) {
+        if ([myError code] != ERROR_NOT_FOUND) {
+            HSLogError(@"error getting salt from target: %@", myError);
+            SETERRORFROMMYERROR;
+            return NO;
+        }
+        
+        
+        // Try to replace salt file with file from cache.
+        
+        saltData = [NSData dataWithContentsOfFile:[self localPath] options:NSUncachedRead error:&myError];
+        if (saltData == nil) {
+            if ([myError code] != ERROR_NOT_FOUND) {
+                HSLogError(@"error reading cached salt file: %@", myError);
+            }
+            SETNSERROR([self errorDomain], -1, @"salt data not found at target or in cache");
+            return NO;
+        }
+        
+        if (![targetConnection setSaltData:saltData forComputerUUID:computerUUID delegate:theDelegate error:error]) {
+            return NO;
+        }
     }
-    return theSalt;
+    return YES;
 }
-            
+- (NSData *)saltDataWithTargetConnectionDelegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    NSData *ret = [NSData dataWithContentsOfFile:[self localPath] options:NSUncachedRead error:error];
+    if (ret == nil) {
+        ret = [self saltFromTargetWithTargetConnectionDelegate:theDelegate error:error];
+    }
+    return ret;
+}
+
 
 #pragma mark internal
 - (NSData *)createRandomSalt {
     unsigned char buf[SALT_LENGTH];
     for (NSUInteger i = 0; i < SALT_LENGTH; i++) {
-        buf[i] = (unsigned char)(rand() % 256);
+        buf[i] = (unsigned char)arc4random_uniform(256);
     }
     return [[[NSData alloc] initWithBytes:buf length:SALT_LENGTH] autorelease];
 }
 - (NSString *)localPath {
-    return [NSString stringWithFormat:@"%@/Cache.noindex/%@/%@/salt.dat", [UserLibrary arqUserLibraryPath], [target targetUUID], computerUUID];
+    return [NSString stringWithFormat:@"%@/%@/%@/salt.dat", [UserLibrary arqCachePath], [target targetUUID], computerUUID];
+}
+- (NSData *)saltFromTargetWithTargetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    NSData *ret = nil;
+    TargetConnection *targetConnection = [target newConnection:error];
+    if (targetConnection == nil) {
+        return nil;
+    }
+    ret = [targetConnection saltDataForComputerUUID:computerUUID delegate:theDelegate error:error];
+    if (ret != nil) {
+        NSError *myError = nil;
+        if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:[self localPath] targetUID:[[CacheOwnership sharedCacheOwnership] uid] targetGID:[[CacheOwnership sharedCacheOwnership] gid] error:&myError]
+            || ![Streams writeData:ret atomicallyToFile:[self localPath] targetUID:[[CacheOwnership sharedCacheOwnership] uid] targetGID:[[CacheOwnership sharedCacheOwnership] gid] bytesWritten:NULL error:&myError]) {
+            HSLogError(@"error caching salt data to %@: %@", [self localPath], myError);
+        }
+    }
+    [targetConnection release];
+    return ret;
 }
 @end

BackupSet.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -32,18 +32,24 @@
 
 
 
+
 @class UserAndComputer;
 @class AppConfig;
 @class Target;
 @protocol TargetConnectionDelegate;
 
 
+@protocol BackupSetActivityListener <NSObject>
+- (void)backupSetActivity:(NSString *)theActivity;
+@end
+
+
 @interface BackupSet : NSObject {
     Target *target;
     NSString *computerUUID;
     UserAndComputer *uac;
 }
-+ (NSArray *)allBackupSetsForTarget:(Target *)theTarget targetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
++ (NSArray *)allBackupSetsForTarget:(Target *)theTarget targetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate activityListener:(id <BackupSetActivityListener>)theActivityListener error:(NSError **)error;
 
 - (id)initWithTarget:(Target *)theTarget
         computerUUID:(NSString *)theComputerUUID

BackupSet.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,51 +31,63 @@
  */
 
 
+
 #import "BackupSet.h"
 #import "S3AuthorizationProvider.h"
 #import "S3Service.h"
 #import "GlacierAuthorizationProvider.h"
 #import "GlacierService.h"
 #import "UserAndComputer.h"
-#import "S3DeleteReceiver.h"
 #import "CryptoKey.h"
 #import "RegexKitLite.h"
 #import "BlobKey.h"
 #import "Commit.h"
 #import "S3ObjectMetadata.h"
-#import "ArqSalt.h"
 #import "AWSRegion.h"
 #import "Bucket.h"
 #import "Target.h"
 #import "TargetConnection.h"
 #import "Repo.h"
+#import "UserLibrary_Arq.h"
+#import "NSString+SBJSON.h"
 
 
 @implementation BackupSet
-+ (NSArray *)allBackupSetsForTarget:(Target *)theTarget targetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
++ (NSArray *)allBackupSetsForTarget:(Target *)theTarget targetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate activityListener:(id<BackupSetActivityListener>)theActivityListener error:(NSError **)error {
-    id <TargetConnection> targetConnection = [[theTarget newConnection] autorelease];
+    TargetConnection *targetConnection = [theTarget newConnection:error];
+    if (targetConnection == nil) {
+        return nil;
+    }
+    NSArray *ret = [BackupSet allBackupSetsForTarget:theTarget targetConnection:targetConnection targetConnectionDelegate:theDelegate activityListener:theActivityListener error:error];
+    [targetConnection release];
+    return ret;
+}
++ (NSArray *)allBackupSetsForTarget:(Target *)theTarget targetConnection:(TargetConnection *)targetConnection targetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate activityListener:(id<BackupSetActivityListener>)theActivityListener error:(NSError **)error {
     NSArray *theComputerUUIDs = [targetConnection computerUUIDsWithDelegate:theDelegate error:error];
     if (theComputerUUIDs == nil) {
         return nil;
     }
-    
+
     NSMutableArray *ret = [NSMutableArray array];
-    for (NSString *theComputerUUID in theComputerUUIDs) {
+    for (NSUInteger i = 0; i < [theComputerUUIDs count]; i++) {
+        [theActivityListener backupSetActivity:[NSString stringWithFormat:@"Loading backup set %ld of %ld at %@", i+1, [theComputerUUIDs count], [theTarget description]]];
+
+        NSString *theComputerUUID = [theComputerUUIDs objectAtIndex:i];
         NSError *uacError = nil;
+        UserAndComputer *uac = nil;
         NSData *uacData = [targetConnection computerInfoForComputerUUID:theComputerUUID delegate:theDelegate error:&uacError];
         if (uacData == nil) {
             HSLogWarn(@"unable to read %@ (skipping): %@", theComputerUUID, [uacError localizedDescription]);
         } else {
-            UserAndComputer *uac = [[[UserAndComputer alloc] initWithXMLData:uacData error:&uacError] autorelease];
+            uac = [[[UserAndComputer alloc] initWithXMLData:uacData error:&uacError] autorelease];
             if (uac == nil) {
                 HSLogError(@"error parsing UserAndComputer data %@: %@", theComputerUUID, uacError);
-            } else {
-                BackupSet *backupSet = [[[BackupSet alloc] initWithTarget:theTarget
-                                                             computerUUID:theComputerUUID
-                                                          userAndComputer:uac] autorelease];
-                [ret addObject:backupSet];
             }
         }
+        BackupSet *backupSet = [[[BackupSet alloc] initWithTarget:theTarget
+                                                     computerUUID:theComputerUUID
+                                                  userAndComputer:uac] autorelease];
+        [ret addObject:backupSet];
     }
     NSSortDescriptor *descriptor = [[[NSSortDescriptor alloc] initWithKey:@"description" ascending:YES] autorelease];
     [ret sortUsingDescriptors:[NSArray arrayWithObject:descriptor]];

BaseTargetConnection.h

@@ -1,73 +0,0 @@
-/*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-@class Target;
-@protocol RemoteFS;
-@protocol TargetConnectionDelegate;
-
-
-@interface BaseTargetConnection : NSObject {
-    Target *target;
-    id <RemoteFS> remoteFS;
-    NSString *pathPrefix;
-}
-
-- (id)initWithTarget:(Target *)theTarget remoteFS:(id <RemoteFS>)theRemoteFS;
-
-- (NSArray *)computerUUIDsWithDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (NSArray *)bucketUUIDsForComputerUUID:(NSString *)theComputerUUID deleted:(BOOL)deleted delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-
-- (NSData *)bucketPlistDataForComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (BOOL)saveBucketPlistData:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (BOOL)deleteBucketPlistDataForComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-
-- (NSData *)computerInfoForComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (BOOL)saveComputerInfo:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-
-- (NSArray *)objectsWithPrefix:(NSString *)thePrefix delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (NSArray *)pathsWithPrefix:(NSString *)thePrefix delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (BOOL)deleteObjectsForComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (BOOL)deletePaths:(NSArray *)thePaths delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-
-- (NSNumber *)fileExistsAtPath:(NSString *)thePath dataSize:(unsigned long long *)theDataSize delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (NSData *)contentsOfFileAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (BOOL)writeData:(NSData *)theData toFileAtPath:(NSString *)thePath dataTransferDelegate:(id <DataTransferDelegate>)theDataTransferDelegate targetConnectionDelegate:(id <TargetConnectionDelegate>)theTargetConnectionDelegate error:(NSError **)error;
-- (BOOL)removeItemAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (NSNumber *)sizeOfItemAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (NSNumber *)isObjectRestoredAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (BOOL)restoreObjectAtPath:(NSString *)thePath forDays:(NSUInteger)theDays alreadyRestoredOrRestoring:(BOOL *)alreadyRestoredOrRestoring delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-
-- (NSData *)saltDataForComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (BOOL)setSaltData:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-
-@end

BaseTargetConnection.m

@@ -1,167 +0,0 @@
-/*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#import "Target.h"
-#import "RemoteFS.h"
-#import "BaseTargetConnection.h"
-#import "RegexKitLite.h"
-#import "TargetConnection.h"
-
-
-@implementation BaseTargetConnection
-- (id)initWithTarget:(Target *)theTarget remoteFS:(id<RemoteFS>)theRemoteFS {
-    if (self = [super init]) {
-        target = [theTarget retain];
-        remoteFS = [theRemoteFS retain];
-        
-        if ([[[theTarget endpoint] path] isEqualToString:@"/"]) {
-            pathPrefix = [@"" retain];
-        } else {
-            pathPrefix = [[[theTarget endpoint] path] retain];
-        }
-    }
-    return self;
-}
-- (void)dealloc {
-    [target release];
-    [remoteFS release];
-    [pathPrefix release];
-    [super dealloc];
-}
-
-- (NSArray *)computerUUIDsWithDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    NSArray *computerUUIDs = [remoteFS contentsOfDirectoryAtPath:[[target endpoint] path] targetConnectionDelegate:theDelegate error:error];
-    if (computerUUIDs == nil) {
-        return nil;
-    }
-    
-    NSMutableArray *ret = [NSMutableArray array];
-    for (NSString *computerUUID in computerUUIDs) {
-        if ([computerUUID rangeOfRegex:@"^(\\S{8}-\\S{4}-\\S{4}-\\S{4}-\\S{12})$"].location != NSNotFound) {
-            [ret addObject:computerUUID];
-        }
-    }
-    return ret;
-}
-
-- (NSArray *)bucketUUIDsForComputerUUID:(NSString *)theComputerUUID deleted:(BOOL)deleted delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    NSString *subdir = deleted ? @"deletedbuckets" : @"buckets";
-    NSString *bucketsPrefix = [NSString stringWithFormat:@"%@/%@/%@/", pathPrefix, theComputerUUID, subdir];
-    NSArray *bucketUUIDs = [remoteFS contentsOfDirectoryAtPath:bucketsPrefix targetConnectionDelegate:theDelegate error:error];
-    if (bucketUUIDs == nil) {
-        return nil;
-    }
-    
-    NSMutableArray *ret = [NSMutableArray array];
-    for (NSString *bucketUUID in bucketUUIDs) {
-        if ([bucketUUID rangeOfRegex:@"^(\\S{8}-\\S{4}-\\S{4}-\\S{4}-\\S{12})$"].location != NSNotFound) {
-            [ret addObject:bucketUUID];
-        }
-    }
-    return ret;
-}
-
-- (NSData *)bucketPlistDataForComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    NSString *subdir = deleted ? @"deletedbuckets" : @"buckets";
-    NSString *path = [NSString stringWithFormat:@"%@/%@/%@/%@", pathPrefix, theComputerUUID, subdir, theBucketUUID];
-    return [remoteFS contentsOfFileAtPath:path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
-}
-- (BOOL)saveBucketPlistData:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    NSString *subdir = deleted ? @"deletedbuckets" : @"buckets";
-    NSString *path = [NSString stringWithFormat:@"%@/%@/%@/%@", pathPrefix, theComputerUUID, subdir, theBucketUUID];
-    return [remoteFS writeData:theData atomicallyToFileAtPath:path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
-}
-- (BOOL)deleteBucketPlistDataForComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    NSString *subdir = deleted ? @"deletedbuckets" : @"buckets";
-    NSString *path = [NSString stringWithFormat:@"%@/%@/%@/%@", pathPrefix, theComputerUUID, subdir, theBucketUUID];
-    return [remoteFS removeItemAtPath:path targetConnectionDelegate:theDelegate error:error];
-}
-
-- (NSData *)computerInfoForComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    NSString *path = [NSString stringWithFormat:@"%@/%@/computerinfo", pathPrefix, theComputerUUID];
-    return [remoteFS contentsOfFileAtPath:path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
-}
-- (BOOL)saveComputerInfo:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    NSString *path = [NSString stringWithFormat:@"%@/%@/computerinfo", pathPrefix, theComputerUUID];
-    return [remoteFS writeData:theData atomicallyToFileAtPath:path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
-}
-
-- (NSArray *)objectsWithPrefix:(NSString *)thePrefix delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [remoteFS objectsAtPath:thePrefix targetConnectionDelegate:theDelegate error:error];
-}
-- (NSArray *)pathsWithPrefix:(NSString *)thePrefix delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [remoteFS pathsOfObjectsAtPath:thePrefix targetConnectionDelegate:theDelegate error:error];
-}
-- (BOOL)deleteObjectsForComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [remoteFS removeItemAtPath:[NSString stringWithFormat:@"%@/%@", pathPrefix, theComputerUUID] targetConnectionDelegate:theDelegate error:error];
-}
-- (BOOL)deletePaths:(NSArray *)thePaths delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    for (NSString *path in thePaths) {
-        if (![remoteFS removeItemAtPath:path targetConnectionDelegate:theDelegate error:error]) {
-            return NO;
-        }
-    }
-    return YES;
-}
-
-- (NSNumber *)fileExistsAtPath:(NSString *)thePath dataSize:(unsigned long long *)theDataSize delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [remoteFS fileExistsAtPath:thePath dataSize:theDataSize targetConnectionDelegate:theDelegate error:error];
-}
-- (NSData *)contentsOfFileAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [remoteFS contentsOfFileAtPath:thePath dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
-}
-- (BOOL)writeData:(NSData *)theData toFileAtPath:(NSString *)thePath dataTransferDelegate:(id <DataTransferDelegate>)theDataTransferDelegate targetConnectionDelegate:(id <TargetConnectionDelegate>)theTargetConnectionDelegate error:(NSError **)error {
-    return [remoteFS writeData:theData atomicallyToFileAtPath:thePath dataTransferDelegate:theDataTransferDelegate targetConnectionDelegate:theTargetConnectionDelegate error:error];
-}
-- (BOOL)removeItemAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [remoteFS removeItemAtPath:thePath targetConnectionDelegate:theDelegate error:error];
-}
-- (NSNumber *)sizeOfItemAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [remoteFS sizeOfItemAtPath:thePath targetConnectionDelegate:theDelegate error:error];
-}
-- (NSNumber *)isObjectRestoredAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [remoteFS isObjectRestoredAtPath:thePath targetConnectionDelegate:theDelegate error:error];
-}
-- (BOOL)restoreObjectAtPath:(NSString *)thePath forDays:(NSUInteger)theDays alreadyRestoredOrRestoring:(BOOL *)alreadyRestoredOrRestoring delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [remoteFS restoreObjectAtPath:thePath forDays:theDays alreadyRestoredOrRestoring:alreadyRestoredOrRestoring targetConnectionDelegate:theDelegate error:error];
-}
-
-- (NSData *)saltDataForComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    NSString *s3Path = [NSString stringWithFormat:@"%@/%@/salt", pathPrefix, theComputerUUID];
-    return [remoteFS contentsOfFileAtPath:s3Path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
-}
-- (BOOL)setSaltData:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    NSString *s3Path = [NSString stringWithFormat:@"%@/%@/salt", pathPrefix, theComputerUUID];
-    return [remoteFS writeData:theData atomicallyToFileAtPath:s3Path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
-}
-@end

BlobKey.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,10 +31,18 @@
  */
 
 
+
 #import "StorageType.h"
 @class BufferedInputStream;
 
 
+typedef enum {
+    BlobKeyCompressionNone = 0,
+    BlobKeyCompressionGzip = 1,
+    BlobKeyCompressionLZ4 = 2
+} BlobKeyCompressionType;
+
+
 @interface BlobKey : NSObject <NSCopying> {
     StorageType storageType;
     NSString *archiveId;
@@ -42,11 +50,12 @@
     NSDate *archiveUploadedDate;
     unsigned char *sha1Bytes;
     BOOL stretchEncryptionKey;
-    BOOL compressed;
+    BlobKeyCompressionType compressionType;
 }
-- (id)initWithSHA1:(NSString *)theSHA1 archiveId:(NSString *)theArchiveId archiveSize:(uint64_t)theArchiveSize archiveUploadedDate:(NSDate *)theArchiveUploadedDate compressed:(BOOL)isCompressed error:(NSError **)error;
+
-- (id)initWithSHA1:(NSString *)theSHA1 storageType:(StorageType)theStorageType stretchEncryptionKey:(BOOL)isStretchedKey compressed:(BOOL)isCompressed error:(NSError **)error;
+- (id)initWithSHA1:(NSString *)theSHA1 archiveId:(NSString *)theArchiveId archiveSize:(uint64_t)theArchiveSize archiveUploadedDate:(NSDate *)theArchiveUploadedDate compressionType:(BlobKeyCompressionType)theCompressionType error:(NSError **)error;
-- (id)initWithStorageType:(StorageType)theStorageType archiveId:(NSString *)theArchiveId archiveSize:(uint64_t)theArchiveSize archiveUploadedDate:(NSDate *)theArchiveUploadedDate sha1:(NSString *)theSHA1 stretchEncryptionKey:(BOOL)isStretchedKey compressed:(BOOL)isCompressed error:(NSError **)error;
+- (id)initWithSHA1:(NSString *)theSHA1 storageType:(StorageType)theStorageType stretchEncryptionKey:(BOOL)isStretchedKey compressionType:(BlobKeyCompressionType)theCompressionType error:(NSError **)error;
+- (id)initWithStorageType:(StorageType)theStorageType archiveId:(NSString *)theArchiveId archiveSize:(uint64_t)theArchiveSize archiveUploadedDate:(NSDate *)theArchiveUploadedDate sha1:(NSString *)theSHA1 stretchEncryptionKey:(BOOL)isStretchedKey compressionType:(BlobKeyCompressionType)theCompressionType error:(NSError **)error;
 - (id)initCopyOfBlobKey:(BlobKey *)theBlobKey withStorageType:(StorageType)theStorageType;
 
 - (StorageType)storageType;
@@ -56,6 +65,6 @@
 - (NSString *)sha1;
 - (unsigned char *)sha1Bytes;
 - (BOOL)stretchEncryptionKey;
-- (BOOL)compressed;
+- (BlobKeyCompressionType)compressionType;
 - (BOOL)isEqualToBlobKey:(BlobKey *)other;
 @end

BlobKey.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,6 +31,7 @@
  */
 
 
+
 #import "BlobKey.h"
 #import "BufferedInputStream.h"
 #import "StringIO.h"
@@ -42,7 +43,7 @@
 
 
 @implementation BlobKey
-- (id)initWithSHA1:(NSString *)theSHA1 archiveId:(NSString *)theArchiveId archiveSize:(uint64_t)theArchiveSize archiveUploadedDate:(NSDate *)theArchiveUploadedDate compressed:(BOOL)isCompressed error:(NSError **)error {
+- (id)initWithSHA1:(NSString *)theSHA1 archiveId:(NSString *)theArchiveId archiveSize:(uint64_t)theArchiveSize archiveUploadedDate:(NSDate *)theArchiveUploadedDate compressionType:(BlobKeyCompressionType)theCompressionType error:(NSError **)error {
     if (self = [super init]) {
         storageType = StorageTypeGlacier;
         
@@ -52,7 +53,7 @@
             return nil;
         }
         if ([sha1Data length] != 20) {
-            SETNSERROR(@"BlobKeyErrorDomain", -1, @"invalid sha1 %@ for BlobKey (must be 20 bytes)", theSHA1);
+            SETNSERROR(@"BlobKeyErrorDomain", -1, @"invalid sha1 '%@' for BlobKey (must be 20 bytes)", theSHA1);
             [self release];
             return nil;
         }
@@ -62,11 +63,11 @@
         archiveId = [theArchiveId retain];
         archiveSize = theArchiveSize;
         archiveUploadedDate = [theArchiveUploadedDate retain];
-        compressed = isCompressed;
+        compressionType = theCompressionType;
     }
     return self;
 }
-- (id)initWithSHA1:(NSString *)theSHA1 storageType:(StorageType)theStorageType stretchEncryptionKey:(BOOL)isStretchedKey compressed:(BOOL)isCompressed error:(NSError **)error {
+- (id)initWithSHA1:(NSString *)theSHA1 storageType:(StorageType)theStorageType stretchEncryptionKey:(BOOL)isStretchedKey compressionType:(BlobKeyCompressionType)theCompressionType error:(NSError **)error {
     if (self = [super init]) {
         storageType = theStorageType;
         
@@ -76,7 +77,7 @@
             return nil;
         }
         if ([sha1Data length] != 20) {
-            SETNSERROR(@"BlobKeyErrorDomain", -1, @"invalid sha1 %@ for BlobKey (must be 20 bytes)", theSHA1);
+            SETNSERROR(@"BlobKeyErrorDomain", -1, @"invalid sha1 '%@' for BlobKey (must be 20 bytes)", theSHA1);
             [self release];
             return nil;
         }
@@ -84,11 +85,11 @@
         memcpy(sha1Bytes, [sha1Data bytes], 20);
         
         stretchEncryptionKey = isStretchedKey;
-        compressed = isCompressed;
+        compressionType = theCompressionType;
     }
     return self;
 }
-- (id)initWithStorageType:(StorageType)theStorageType archiveId:(NSString *)theArchiveId archiveSize:(uint64_t)theArchiveSize archiveUploadedDate:(NSDate *)theArchiveUploadedDate sha1:(NSString *)theSHA1 stretchEncryptionKey:(BOOL)isStretchedKey compressed:(BOOL)isCompressed error:(NSError **)error {
+- (id)initWithStorageType:(StorageType)theStorageType archiveId:(NSString *)theArchiveId archiveSize:(uint64_t)theArchiveSize archiveUploadedDate:(NSDate *)theArchiveUploadedDate sha1:(NSString *)theSHA1 stretchEncryptionKey:(BOOL)isStretchedKey compressionType:(BlobKeyCompressionType)theCompressionType error:(NSError **)error {
     if (self = [super init]) {
         storageType = theStorageType;
         archiveId = [theArchiveId retain];
@@ -101,7 +102,7 @@
             return nil;
         }
         if ([sha1Data length] != 20) {
-            SETNSERROR(@"BlobKeyErrorDomain", -1, @"invalid sha1 %@ for BlobKey (must be 20 bytes)", theSHA1);
+            SETNSERROR(@"BlobKeyErrorDomain", -1, @"invalid sha1 '%@' for BlobKey (must be 20 bytes)", theSHA1);
             [self release];
             return nil;
         }
@@ -109,7 +110,7 @@
         memcpy(sha1Bytes, [sha1Data bytes], 20);
         
         stretchEncryptionKey = isStretchedKey;
-        compressed = isCompressed;
+        compressionType = theCompressionType;
     }
     return self;
 }
@@ -120,7 +121,7 @@
                             archiveUploadedDate:[theBlobKey archiveUploadedDate]
                                       sha1Bytes:[theBlobKey sha1Bytes]
                            stretchEncryptionKey:[theBlobKey stretchEncryptionKey]
-                                     compressed:[theBlobKey compressed]];
+                                compressionType:[theBlobKey compressionType]];
 }
 - (void)dealloc {
     [archiveId release];
@@ -150,8 +151,8 @@
 - (BOOL)stretchEncryptionKey {
     return stretchEncryptionKey;
 }
-- (BOOL)compressed {
+- (BlobKeyCompressionType)compressionType {
-    return compressed;
+    return compressionType;
 }
 - (BOOL)isEqualToBlobKey:(BlobKey *)other {
     if (memcmp(sha1Bytes, [other sha1Bytes], 20) != 0) {
@@ -166,17 +167,17 @@
 
 #pragma mark NSCopying
 - (id)copyWithZone:(NSZone *)zone {
-    return [[BlobKey alloc] initWithStorageType:storageType archiveId:archiveId archiveSize:archiveSize archiveUploadedDate:archiveUploadedDate sha1Bytes:sha1Bytes stretchEncryptionKey:stretchEncryptionKey compressed:compressed];
+    return [[BlobKey alloc] initWithStorageType:storageType archiveId:archiveId archiveSize:archiveSize archiveUploadedDate:archiveUploadedDate sha1Bytes:sha1Bytes stretchEncryptionKey:stretchEncryptionKey compressionType:compressionType];
 }
 
 
 #pragma mark NSObject
 - (NSString *)description {
     if (storageType == StorageTypeS3 || storageType == StorageTypeS3Glacier) {
-        NSString *type = storageType == StorageTypeS3 ? @"S3" : @"S3Glacier";
+        NSString *type = storageType == StorageTypeS3 ? @"Standard" : @"S3Glacier";
-        return [NSString stringWithFormat:@"<BlobKey sha1=%@,type=%@,stretchedkey=%@,compressed=%@>", [self sha1], type, (stretchEncryptionKey ? @"YES" : @"NO"), (compressed ? @"YES" : @"NO")];
+        return [NSString stringWithFormat:@"<BlobKey sha1=%@,type=%@,stretchedkey=%@,compression=%d>", [self sha1], type, (stretchEncryptionKey ? @"YES" : @"NO"), compressionType];
     }
-    return [NSString stringWithFormat:@"<BlobKey sha1=%@,type=Glacier,archiveId=%@,archiveSize=%qu,archiveUploadedDate=%@,stretchedkey=%@,compressed=%@>", [self sha1], archiveId, archiveSize, [self archiveUploadedDate], (stretchEncryptionKey ? @"YES" : @"NO"), (compressed ? @"YES" : @"NO")];
+    return [NSString stringWithFormat:@"<BlobKey sha1=%@,type=Glacier,archiveId=%@,archiveSize=%qu,archiveUploadedDate=%@,stretchedkey=%@,compression=%d>", [self sha1], archiveId, archiveSize, [self archiveUploadedDate], (stretchEncryptionKey ? @"YES" : @"NO"), compressionType];
 }
 - (BOOL)isEqual:(id)anObject {
     if (![anObject isKindOfClass:[BlobKey class]]) {
@@ -190,7 +191,7 @@
     && [NSObject equalObjects:archiveId and:[other archiveId]]
     && archiveSize == [other archiveSize]
     && [NSObject equalObjects:archiveUploadedDate and:[other archiveUploadedDate]]
-    && compressed == [other compressed];
+    && compressionType == [other compressionType];
 }
 - (NSUInteger)hash {
     return (NSUInteger)(*sha1Bytes);
@@ -198,7 +199,7 @@
 
 
 #pragma mark internal
-- (id)initWithStorageType:(StorageType)theStorageType archiveId:(NSString *)theArchiveId archiveSize:(uint64_t)theArchiveSize archiveUploadedDate:(NSDate *)theArchiveUploadedDate sha1Bytes:(unsigned char *)theSHA1Bytes stretchEncryptionKey:(BOOL)isStretchedKey compressed:(BOOL)isCompressed {
+- (id)initWithStorageType:(StorageType)theStorageType archiveId:(NSString *)theArchiveId archiveSize:(uint64_t)theArchiveSize archiveUploadedDate:(NSDate *)theArchiveUploadedDate sha1Bytes:(unsigned char *)theSHA1Bytes stretchEncryptionKey:(BOOL)isStretchedKey compressionType:(BlobKeyCompressionType)theCompressionType {
     if (self = [super init]) {
         storageType = theStorageType;
         archiveId = [theArchiveId retain];
@@ -210,7 +211,7 @@
         memcpy(sha1Bytes, theSHA1Bytes, 20);
         
         stretchEncryptionKey = isStretchedKey;
-        compressed = isCompressed;
+        compressionType = theCompressionType;
     }
     return self;
 }

BlobKeyIO.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,9 +31,10 @@
  */
 
 
+
 @class BufferedInputStream;
 @class BufferedOutputStream;
-@class BlobKey;
+#import "BlobKey.h"
 
 
 @interface BlobKeyIO : NSObject {
@@ -41,5 +42,5 @@
 }
 + (void)write:(BlobKey *)theBlobKey to:(NSMutableData *)data;
 + (BOOL)write:(BlobKey *)theBlobKey to:(BufferedOutputStream *)os error:(NSError **)error;
-+ (BOOL)read:(BlobKey **)theBlobKey from:(BufferedInputStream *)is treeVersion:(int)theTreeVersion compressed:(BOOL)isCompressed error:(NSError **)error;
++ (BOOL)read:(BlobKey **)theBlobKey from:(BufferedInputStream *)is treeVersion:(int)theTreeVersion compressionType:(BlobKeyCompressionType)theCompressionType error:(NSError **)error;
 @end

BlobKeyIO.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,6 +31,7 @@
  */
 
 
+
 #import "BlobKeyIO.h"
 #import "BooleanIO.h"
 #import "StringIO.h"
@@ -57,7 +58,7 @@
     && [IntegerIO writeUInt64:[theBlobKey archiveSize] to:os error:error]
     && [DateIO write:[theBlobKey archiveUploadedDate] to:os error:error];
 }
-+ (BOOL)read:(BlobKey **)theBlobKey from:(BufferedInputStream *)is treeVersion:(int)theTreeVersion compressed:(BOOL)isCompressed error:(NSError **)error {
++ (BOOL)read:(BlobKey **)theBlobKey from:(BufferedInputStream *)is treeVersion:(int)theTreeVersion compressionType:(BlobKeyCompressionType)theCompressionType error:(NSError **)error {
     NSString *dataSHA1;
     BOOL stretchEncryptionKey = NO;
     StorageType storageType = StorageTypeS3;
@@ -86,7 +87,7 @@
         // If the sha1 is nil, it must have been a nil BlobKey, so we return nil here.
         *theBlobKey = nil;
     } else {
-        *theBlobKey = [[[BlobKey alloc] initWithStorageType:storageType archiveId:archiveId archiveSize:archiveSize archiveUploadedDate:archiveUploadedDate sha1:dataSHA1 stretchEncryptionKey:stretchEncryptionKey compressed:isCompressed error:error] autorelease];
+        *theBlobKey = [[[BlobKey alloc] initWithStorageType:storageType archiveId:archiveId archiveSize:archiveSize archiveUploadedDate:archiveUploadedDate sha1:dataSHA1 stretchEncryptionKey:stretchEncryptionKey compressionType:theCompressionType error:error] autorelease];
         if (*theBlobKey == nil) {
             return NO;
         }

Bucket.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,6 +31,8 @@
  */
 
 
+
+
 #import "StorageType.h"
 @class AWSRegion;
 @class DictNode;
@@ -40,8 +42,13 @@
 @class BufferedInputStream;
 @class BufferedOutputStream;
 @protocol TargetConnectionDelegate;
+@class TargetConnection;
 
 
+@protocol BucketActivityListener <NSObject>
+- (void)bucketActivity:(NSString *)theActivity;
+@end
+
 enum {
     BucketPathMixedState = -1,
     BucketPathOffState   =  0,
@@ -59,10 +66,12 @@ typedef NSInteger BucketPathState;
     StorageType storageType;
     NSMutableArray *ignoredRelativePaths;
     BucketExcludeSet *excludeSet;
-    NSMutableArray *stringArrayPairs;
     NSString *vaultName;
     NSDate *vaultCreatedDate;
     NSDate *plistDeletedDate;
+    BOOL skipDuringBackup;
+    BOOL excludeItemsWithTimeMachineExcludeMetadataFlag;
+    BOOL skipIfNotMounted;
 }
 
 + (NSArray *)bucketsWithTarget:(Target *)theTarget
@@ -71,18 +80,18 @@ typedef NSInteger BucketPathState;
       targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD
                          error:(NSError **)error;
 
++ (NSArray *)bucketsWithTarget:(Target *)theTarget
+                  computerUUID:(NSString *)theComputerUUID
+            encryptionPassword:(NSString *)theEncryptionPassword
+      targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD
+              activityListener:(id <BucketActivityListener>)theActivityListener
+                         error:(NSError **)error;
+
 + (NSArray *)bucketUUIDsWithTarget:(Target *)theTarget
                       computerUUID:(NSString *)theComputerUUID
-                encryptionPassword:(NSString *)theEncryptionPassword
           targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD
                              error:(NSError **)error;
 
-+ (NSArray *)deletedBucketsWithTarget:(Target *)theTarget
-                         computerUUID:(NSString *)theComputerUUID
-                   encryptionPassword:(NSString *)theEncryptionPassword
-             targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD
-                                error:(NSError **)error;
-
 + (NSString *)errorDomain;
 
 - (id)initWithTarget:(Target *)theTarget
@@ -106,10 +115,12 @@ typedef NSInteger BucketPathState;
 - (NSString *)vaultName;
 - (NSDate *)vaultCreatedDate;
 - (NSDate *)plistDeletedDate;
+- (BOOL)skipDuringBackup;
+- (BOOL)excludeItemsWithTimeMachineExcludeMetadataFlag;
 - (BucketPathState)stateForPath:(NSString *)thePath ignoreExcludes:(BOOL)ignoreExcludes;
-- (void)setIgnoredRelativePaths:(NSSet *)theSet;
 - (NSSet *)ignoredRelativePaths;
-- (void)enteredPath:(NSString *)thePath;
+- (BOOL)skipIfNotMounted;
-- (void)leftPath:(NSString *)thePath;
 - (NSData *)toXMLData;
+- (BOOL)writeTo:(BufferedOutputStream *)theBOS error:(NSError **)error;
+- (void)writeTo:(NSMutableData *)data;
 @end

Bucket.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,29 +31,33 @@
  */
 
 
+
 #import "Bucket.h"
 #import "DictNode.h"
 #import "ArrayNode.h"
 #import "StringNode.h"
+#import "BooleanNode.h"
 #import "NSString_slashed.h"
 #import "BucketExcludeSet.h"
 #import "S3AuthorizationProvider.h"
 #import "S3Service.h"
 #import "UserLibrary_Arq.h"
 #import "S3Service.h"
+#import "FSStat.h"
+#import "Volume.h"
 #import "StorageType.h"
 #import "GlacierService.h"
 #import "AWSRegion.h"
 #import "RegexKitLite.h"
 #import "AWSRegion.h"
-#import "CryptoKey.h"
+#import "ObjectEncryptor.h"
 #import "Target.h"
 #import "TargetConnection.h"
 #import "GlacierAuthorizationProvider.h"
 #import "GlacierService.h"
-#import "ArqSalt.h"
 #import "DataIO.h"
 #import "StringIO.h"
+#import "NSString_extra.h"
 
 
 #define BUCKET_PLIST_SALT "BucketPL"
@@ -98,9 +102,28 @@
             encryptionPassword:(NSString *)theEncryptionPassword
       targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD
                          error:(NSError **)error {
-    id <TargetConnection> targetConnection = [theTarget newConnection];
+    return [Bucket bucketsWithTarget:theTarget
+                        computerUUID:theComputerUUID
+                  encryptionPassword:(NSString *)theEncryptionPassword
+            targetConnectionDelegate:theTCD
+                    activityListener:nil
+                               error:error];
+}
++ (NSArray *)bucketsWithTarget:(Target *)theTarget
+                  computerUUID:(NSString *)theComputerUUID
+            encryptionPassword:(NSString *)theEncryptionPassword
+      targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD
+              activityListener:(id <BucketActivityListener>)theActivityListener
+                         error:(NSError **)error {
+    HSLogDebug(@"bucketsWithTarget: theTarget=%@ endpoint=%@ path=%@", theTarget, [theTarget endpoint], [[theTarget endpoint] path]);
+    
+    TargetConnection *targetConnection = [theTarget newConnection:error];
+    if (targetConnection == nil) {
+        return nil;
+    }
     NSMutableArray *ret = [NSMutableArray array];
     do {
+        [theActivityListener bucketActivity:@"Loading folder list..."];
         NSArray *bucketUUIDs = [targetConnection bucketUUIDsForComputerUUID:theComputerUUID deleted:NO delegate:theTCD error:error];
         if (bucketUUIDs == nil) {
             if (error != NULL) {
@@ -109,9 +132,18 @@
             ret = nil;
             break;
         }
-        for (NSString *bucketUUID in bucketUUIDs) {
+        for (NSUInteger i = 0; i < [bucketUUIDs count]; i++) {
+            [theActivityListener bucketActivity:[NSString stringWithFormat:@"Loading folder %ld of %ld", i+1, [bucketUUIDs count]]];
+            
+            NSString *bucketUUID = [bucketUUIDs objectAtIndex:i];
             NSError *myError = nil;
-            Bucket *bucket = [Bucket bucketWithTarget:theTarget targetConnection:targetConnection computerUUID:theComputerUUID bucketUUID:bucketUUID encryptionPassword:theEncryptionPassword error:&myError];
+            Bucket *bucket = [Bucket bucketWithTarget:theTarget
+                                     targetConnection:targetConnection
+                                         computerUUID:theComputerUUID
+                                   encryptionPassword:theEncryptionPassword
+                                           bucketUUID:bucketUUID
+                             targetConnectionDelegate:theTCD
+                                                error:&myError];
             if (bucket == nil) {
                 HSLogError(@"failed to load bucket plist for %@/%@: %@", theComputerUUID, bucketUUID, myError);
                 if ([myError code] != ERROR_INVALID_PLIST_XML) {
@@ -127,61 +159,23 @@
         [ret sortUsingDescriptors:[NSArray arrayWithObject:descriptor]];
     } while(0);
     [targetConnection release];
+    HSLogDebug(@"returning %ld buckets for computer %@", [ret count], theComputerUUID);
     return ret;
 }
 + (NSArray *)bucketUUIDsWithTarget:(Target *)theTarget
                       computerUUID:(NSString *)theComputerUUID
-                encryptionPassword:(NSString *)theEncryptionPassword
           targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD
                              error:(NSError **)error {
-    id <TargetConnection> targetConnection = [[theTarget newConnection] autorelease];
+    TargetConnection *targetConnection = [theTarget newConnection:error];
-    return [targetConnection bucketUUIDsForComputerUUID:theComputerUUID deleted:NO delegate:theTCD error:error];
+    if (targetConnection == nil) {
-}
-+ (NSArray *)deletedBucketsWithTarget:(Target *)theTarget
-                         computerUUID:(NSString *)theComputerUUID
-                   encryptionPassword:(NSString *)theEncryptionPassword
-             targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD
-                                error:(NSError **)error {
-    HSLogDebug(@"deletedBucketsWithTarget: theTarget=%@ endpoint=%@ path=%@", theTarget, [theTarget endpoint], [[theTarget endpoint] path]);
-    
-    NSData *salt = [NSData dataWithBytes:BUCKET_PLIST_SALT length:8];
-    CryptoKey *cryptoKey = [[[CryptoKey alloc] initWithPassword:theEncryptionPassword salt:salt error:error] autorelease];
-    if (cryptoKey == nil) {
         return nil;
     }
-    
+    NSArray *ret = [targetConnection bucketUUIDsForComputerUUID:theComputerUUID deleted:NO delegate:theTCD error:error];
-    NSMutableArray *ret = [NSMutableArray array];
+    [targetConnection release];
-    id <TargetConnection> targetConnection = [[theTarget newConnection] autorelease];
-    NSArray *deletedBucketUUIDs = [targetConnection bucketUUIDsForComputerUUID:theComputerUUID deleted:YES delegate:theTCD error:error];
-    if (deletedBucketUUIDs == nil) {
-        return nil;
-    }
-    for (NSString *bucketUUID in deletedBucketUUIDs) {
-        NSData *data = [targetConnection bucketPlistDataForComputerUUID:theComputerUUID bucketUUID:bucketUUID deleted:YES delegate:theTCD error:error];
-        if (data == nil) {
-            return nil;
-        }
-        if (!strncmp([data bytes], "encrypted", 9)) {
-            NSData *encryptedData = [data subdataWithRange:NSMakeRange(9, [data length] - 9)];
-            data = [cryptoKey decrypt:encryptedData error:error];
-            if (data == nil) {
-                return nil;
-            }
-        }
-        NSError *myError = nil;
-        DictNode *plist = [DictNode dictNodeWithXMLData:data error:&myError];
-        if (plist == nil) {
-            SETNSERROR(@"BucketErrorDomain", -1, @"error parsing bucket plist %@ %@: %@", theComputerUUID, bucketUUID, [myError localizedDescription]);
-            return nil;
-        }
-        Bucket *bucket = [[[Bucket alloc] initWithTarget:theTarget plist:plist] autorelease];
-        [ret addObject:bucket];
-    }
-    NSSortDescriptor *descriptor = [[[NSSortDescriptor alloc] initWithKey:@"bucketName" ascending:YES selector:@selector(caseInsensitiveCompare:)] autorelease];
-    [ret sortUsingDescriptors:[NSArray arrayWithObject:descriptor]];
     return ret;
 }
 
+
 + (NSString *)errorDomain {
     return @"BucketErrorDomain";
 }
@@ -203,7 +197,7 @@
         storageType = theStorageType;
         ignoredRelativePaths = [[NSMutableArray alloc] init];
         excludeSet = [[BucketExcludeSet alloc] init];
-        stringArrayPairs = [[NSMutableArray alloc] init];
+        excludeItemsWithTimeMachineExcludeMetadataFlag = NO; // Default to false because things might get unexpectedly skipped, e.g. VMWare VMs
     }
     return self;
 }
@@ -239,7 +233,6 @@
     [localMountPoint release];
     [ignoredRelativePaths release];
     [excludeSet release];
-    [stringArrayPairs release];
     [vaultName release];
     [vaultCreatedDate release];
     [plistDeletedDate release];
@@ -279,24 +272,34 @@
 - (NSDate *)plistDeletedDate {
     return plistDeletedDate;
 }
+- (BOOL)skipDuringBackup {
+    return skipDuringBackup;
+}
+- (BOOL)excludeItemsWithTimeMachineExcludeMetadataFlag {
+    return excludeItemsWithTimeMachineExcludeMetadataFlag;
+}
 - (BucketPathState)stateForPath:(NSString *)thePath ignoreExcludes:(BOOL)ignoreExcludes {
     if ([ignoredRelativePaths containsObject:@""]) {
         return BucketPathOffState;
     }
     
     NSInteger ret = BucketPathOnState;
-    NSString *relativePath = [thePath substringFromIndex:[localPath length]];
+    if ([thePath length] <= [localPath length]) {
-    for (NSString *ignoredRelativePath in ignoredRelativePaths) {
+        HSLogDebug(@"path %@ isn't longer than localPath %@", thePath, localPath);
-        if ([relativePath isEqualToString:ignoredRelativePath]
+    } else {
-            || ([relativePath hasPrefix:ignoredRelativePath] && ([relativePath length] > [ignoredRelativePath length]) && ([relativePath characterAtIndex:[ignoredRelativePath length]] == '/'))
+        NSString *relativePath = [thePath substringFromIndex:[localPath length]];
-            ) {
+        for (NSString *ignoredRelativePath in ignoredRelativePaths) {
-            ret = BucketPathOffState;
+            if ([relativePath isEqualToString:ignoredRelativePath]
-            break;
+                || ([relativePath hasPrefix:ignoredRelativePath] && ([relativePath length] > [ignoredRelativePath length]) && ([relativePath characterAtIndex:[ignoredRelativePath length]] == '/'))
-        } else if (([ignoredRelativePath hasPrefix:relativePath] || [relativePath length] == 0)
+                ) {
-                   && ([ignoredRelativePath length] > [relativePath length])
+                ret = BucketPathOffState;
-                   && ([relativePath isEqualToString:@""] || [relativePath isEqualToString:@"/"] || [ignoredRelativePath characterAtIndex:[relativePath length]] == '/')) {
+                break;
-            ret = BucketPathMixedState;
+            } else if (([ignoredRelativePath hasPrefix:relativePath] || [relativePath length] == 0)
-            break;
+                       && ([ignoredRelativePath length] > [relativePath length])
+                       && ([relativePath isEqualToString:@""] || [relativePath isEqualToString:@"/"] || [ignoredRelativePath characterAtIndex:[relativePath length]] == '/')) {
+                ret = BucketPathMixedState;
+                break;
+            }
         }
     }
     if (!ignoreExcludes && [excludeSet matchesFullPath:thePath filename:[thePath lastPathComponent]]) {
@@ -304,38 +307,11 @@
     }
     return ret;
 }
-- (void)setIgnoredRelativePaths:(NSSet *)theSet {
-    [ignoredRelativePaths setArray:[theSet allObjects]];
-}
 - (NSSet *)ignoredRelativePaths {
     return [NSSet setWithArray:ignoredRelativePaths];
 }
-- (void)enteredPath:(NSString *)thePath {
+- (BOOL)skipIfNotMounted {
-    NSMutableArray *relativePathsToSkip = [NSMutableArray array];
+    return skipIfNotMounted;
-    if (![thePath isEqualToString:localPath]) {
-        NSString *relativePath = [thePath substringFromIndex:[localPath length]];
-        for (NSString *ignored in ignoredRelativePaths) {
-            BOOL applicable = NO;
-            if ([ignored hasPrefix:relativePath]) {
-                if ([ignored isEqualToString:relativePath] || ([ignored characterAtIndex:[relativePath length]] == '/')) {
-                    applicable = YES;
-                }
-            }
-            if (!applicable) {
-                [relativePathsToSkip addObject:ignored];
-            }
-        }
-    }
-    StringArrayPair *sap = [[StringArrayPair alloc] initWithLeft:thePath right:relativePathsToSkip];
-    [stringArrayPairs addObject:sap];
-    [sap release];
-    [ignoredRelativePaths removeObjectsInArray:relativePathsToSkip];
-}
-- (void)leftPath:(NSString *)thePath {
-    StringArrayPair *sap = [stringArrayPairs lastObject];
-    NSAssert([[sap left] isEqualToString:thePath], @"must leave last path on the stack!");
-    [ignoredRelativePaths addObjectsFromArray:[sap right]];
-    [stringArrayPairs removeLastObject];
 }
 - (NSData *)toXMLData {
     DictNode *plist = [[[DictNode alloc] init] autorelease];
@@ -346,13 +322,17 @@
     [plist putString:localPath forKey:@"LocalPath"];
     [plist putString:localMountPoint forKey:@"LocalMountPoint"];
     [plist putInt:storageType forKey:@"StorageType"];
-    [plist putString:vaultName forKey:@"VaultName"];
+    if (vaultName != nil) {
+        [plist putString:vaultName forKey:@"VaultName"];
+    }
     if (vaultCreatedDate != nil) {
         [plist putDouble:[vaultCreatedDate timeIntervalSinceReferenceDate] forKey:@"VaultCreatedTime"];
     }
     if (plistDeletedDate != nil) {
         [plist putDouble:[plistDeletedDate timeIntervalSinceReferenceDate] forKey:@"PlistDeletedTime"];
     }
+    [plist putBoolean:skipDuringBackup forKey:@"SkipDuringBackup"];
+    [plist putBoolean:excludeItemsWithTimeMachineExcludeMetadataFlag forKey:@"ExcludeItemsWithTimeMachineExcludeMetadataFlag"];
     ArrayNode *ignoredRelativePathsNode = [[[ArrayNode alloc] init] autorelease];
     [plist put:ignoredRelativePathsNode forKey:@"IgnoredRelativePaths"];
     NSArray *sortedRelativePaths = [ignoredRelativePaths sortedArrayUsingSelector:@selector(caseInsensitiveCompare:)];
@@ -360,14 +340,21 @@
         [ignoredRelativePathsNode add:[[[StringNode alloc] initWithString:ignoredRelativePath] autorelease]];
     }
     [plist put:[excludeSet toPlist] forKey:@"Excludes"];
+    [plist putBoolean:skipIfNotMounted forKey:@"SkipIfNotMounted"];
     return [plist XMLData];
 }
-
+- (BOOL)writeTo:(BufferedOutputStream *)theBOS error:(NSError **)error {
+    NSData *data = [self toXMLData];
+    return [target writeTo:theBOS error:error] && [DataIO write:data to:theBOS error:error];
+}
+- (void)writeTo:(NSMutableData *)data {
+    [target writeTo:data];
+    [DataIO write:[self toXMLData] to:data];
+}
 
 #pragma mark NSCopying
 - (id)copyWithZone:(NSZone *)zone {
     BucketExcludeSet *excludeSetCopy = [excludeSet copyWithZone:zone];
-    NSMutableArray *stringArrayPairsCopy = [stringArrayPairs copyWithZone:zone];
     NSMutableArray *ignoredRelativePathsCopy = [[NSMutableArray alloc] initWithArray:ignoredRelativePaths copyItems:YES];
     Bucket *ret = [[Bucket alloc] initWithTarget:target
                                       bucketUUID:bucketUUID
@@ -378,37 +365,44 @@
                                      storageType:storageType
                             ignoredRelativePaths:ignoredRelativePathsCopy
                                       excludeSet:excludeSetCopy
-                                stringArrayPairs:stringArrayPairsCopy
                                        vaultName:vaultName
                                 vaultCreatedDate:vaultCreatedDate
-                                plistDeletedDate:plistDeletedDate];
+                                plistDeletedDate:plistDeletedDate
+                                skipDuringBackup:skipDuringBackup
+  excludeItemsWithTimeMachineExcludeMetadataFlag:excludeItemsWithTimeMachineExcludeMetadataFlag
+                                skipIfNotMounted:skipIfNotMounted];
     [excludeSetCopy release];
-    [stringArrayPairsCopy release];
     [ignoredRelativePathsCopy release];
     return ret;
 }
 
 #pragma mark NSObject
 - (NSString *)description {
-    return [NSString stringWithFormat:@"<Bucket %@: %@ (%lu ignored paths)>", bucketUUID, localPath, (unsigned long)[ignoredRelativePaths count]];
+    NSUInteger ignoredCount = [ignoredRelativePaths count];
+    return [NSString stringWithFormat:@"<Bucket %@: %@ (%lu ignored path%@)>", bucketUUID, localPath, (unsigned long)ignoredCount, (ignoredCount == 1 ? @"" : @"s")];
 }
 
 
 #pragma mark internal
 + (Bucket *)bucketWithTarget:(Target *)theTarget
-            targetConnection:(id <TargetConnection>)theTargetConnection
+            targetConnection:(TargetConnection *)theTargetConnection
                 computerUUID:(NSString *)theComputerUUID
-                  bucketUUID:(NSString *)theBucketUUID
           encryptionPassword:(NSString *)theEncryptionPassword
+                  bucketUUID:(NSString *)theBucketUUID
+    targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD
                        error:(NSError **)error {
-    NSData *salt = [NSData dataWithBytes:BUCKET_PLIST_SALT length:8];
+    ObjectEncryptor *encryptor = [[[ObjectEncryptor alloc] initWithTarget:theTarget
-    CryptoKey *cryptoKey = [[[CryptoKey alloc] initWithPassword:theEncryptionPassword salt:salt error:error] autorelease];
+                                                             computerUUID:theComputerUUID
-    if (cryptoKey == nil) {
+                                                       encryptionPassword:theEncryptionPassword
+                                                             customV1Salt:[NSData dataWithBytes:BUCKET_PLIST_SALT length:strlen(BUCKET_PLIST_SALT)]
+                                                 targetConnectionDelegate:nil
+                                                                    error:error] autorelease];
+    if (encryptor == nil) {
         return nil;
     }
-    
+
     BOOL encrypted = NO;
-    NSData *data = [theTargetConnection bucketPlistDataForComputerUUID:theComputerUUID bucketUUID:theBucketUUID deleted:NO delegate:nil error:error];
+    NSData *data = [theTargetConnection bucketPlistDataForComputerUUID:theComputerUUID bucketUUID:theBucketUUID deleted:NO delegate:theTCD error:error];
     if (data == nil) {
         return nil;
     }
@@ -419,7 +413,7 @@
     if (length >= 9 && !strncmp([data bytes], "encrypted", length)) {
         encrypted = YES;
         NSData *encryptedData = [data subdataWithRange:NSMakeRange(9, [data length] - 9)];
-        data = [cryptoKey decrypt:encryptedData error:error];
+        data = [encryptor decryptedDataForObject:encryptedData error:error];
         if (data == nil) {
             return nil;
         }
@@ -456,6 +450,12 @@
         if ([thePlist containsKey:@"PlistDeletedTime"]) {
             plistDeletedDate = [[NSDate alloc] initWithTimeIntervalSinceReferenceDate:[[thePlist realNodeForKey:@"PlistDeletedTime"] doubleValue]];
         }
+        if ([thePlist containsKey:@"SkipDuringBackup"]){
+            skipDuringBackup = [[thePlist booleanNodeForKey:@"SkipDuringBackup"] booleanValue];
+        }
+        if ([thePlist containsKey:@"ExcludeItemsWithTimeMachineExcludeMetadataFlag"]) {
+            excludeItemsWithTimeMachineExcludeMetadataFlag = [[thePlist booleanNodeForKey:@"ExcludeItemsWithTimeMachineExcludeMetadataFlag"] booleanValue];
+        }
         ignoredRelativePaths = [[NSMutableArray alloc] init];
         ArrayNode *ignoredPathsNode = [thePlist arrayNodeForKey:@"IgnoredRelativePaths"];
         for (NSUInteger index = 0; index < [ignoredPathsNode size]; index++) {
@@ -464,7 +464,10 @@
         [self sortIgnoredRelativePaths];
         excludeSet = [[BucketExcludeSet alloc] init];
         [excludeSet loadFromPlist:[thePlist dictNodeForKey:@"Excludes"] localPath:localPath];
-        stringArrayPairs = [[NSMutableArray alloc] init];
+        
+        if ([thePlist containsKey:@"SkipIfNotMounted"]) {
+            skipIfNotMounted = [[thePlist booleanNodeForKey:@"SkipIfNotMounted"] booleanValue];
+        }
     }
     return self;
 }
@@ -475,6 +478,9 @@
     [ignoredRelativePaths setArray:[set allObjects]];
     [ignoredRelativePaths sortUsingSelector:@selector(compareByLength:)];
 }
+
+
+
 - (id)initWithTarget:(Target *)theTarget
           bucketUUID:(NSString *)theBucketUUID
           bucketName:(NSString *)theBucketName
@@ -484,10 +490,12 @@
          storageType:(int)theStorageType
 ignoredRelativePaths:(NSMutableArray *)theIgnoredRelativePaths
           excludeSet:(BucketExcludeSet *)theExcludeSet
-    stringArrayPairs:(NSMutableArray *)theStringArrayPairs
            vaultName:(NSString *)theVaultName
     vaultCreatedDate:(NSDate *)theVaultCreatedDate
-    plistDeletedDate:(NSDate *)thePlistDeletedDate {
+    plistDeletedDate:(NSDate *)thePlistDeletedDate
+    skipDuringBackup:(BOOL)theSkipDuringBackup
+excludeItemsWithTimeMachineExcludeMetadataFlag:(BOOL)theExcludeItemsWithTimeMachineExcludeMetadataFlag
+    skipIfNotMounted:(BOOL)theSkipIfNotMounted {
     if (self = [super init]) {
         target = [theTarget retain];
         bucketUUID = [theBucketUUID retain];
@@ -498,10 +506,12 @@ ignoredRelativePaths:(NSMutableArray *)theIgnoredRelativePaths
         storageType = theStorageType;
         ignoredRelativePaths = [theIgnoredRelativePaths retain];
         excludeSet = [theExcludeSet retain];
-        stringArrayPairs = [theStringArrayPairs retain];
         vaultName = [theVaultName retain];
         vaultCreatedDate = [theVaultCreatedDate retain];
         plistDeletedDate = [thePlistDeletedDate retain];
+        skipDuringBackup = theSkipDuringBackup;
+        excludeItemsWithTimeMachineExcludeMetadataFlag = theExcludeItemsWithTimeMachineExcludeMetadataFlag;
+        skipIfNotMounted = theSkipIfNotMounted;
     }
     return self;
 }

BucketExclude.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -32,6 +32,8 @@
 
 
 
+
+
 @class DictNode;
 
 enum {

BucketExclude.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,6 +31,8 @@
  */
 
 
+
+
 #import "BucketExclude.h"
 #import "DictNode.h"
 #import "IntegerNode.h"

BucketExcludeSet.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -32,6 +32,8 @@
 
 
 
+
+
 #import "BucketExclude.h"
 @class DictNode;
 

BucketExcludeSet.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,6 +31,8 @@
  */
 
 
+
+
 #import "BucketExcludeSet.h"
 #import "BucketExclude.h"
 #import "DictNode.h"

ByteSize.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -32,12 +32,11 @@
 
 
 
-#import "S3Receiver.h"
-@class S3Service;
 
-@interface S3DeleteReceiver : NSObject <S3Receiver> {
+
-    S3Service *s3;
+
+@interface ByteSize : NSObject {
+
 }
-- (id)initWithS3Service:(S3Service *)theS3;
++ (NSString *)descriptionForSize:(unsigned long long)value;
-
 @end

ByteSize.m

@@ -0,0 +1,53 @@
+/*
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
+ 
+ All rights reserved.
+ 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ 
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 
+ * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
+ their contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+#import "ByteSize.h"
+
+
+@implementation ByteSize
++ (NSString *)descriptionForSize:(unsigned long long)value {
+    if (value > 1000000000L) {
+        double d = ((double)value) / 1000000000.0;
+        return [NSString stringWithFormat:@"%0.3f GB", d];
+    } else if (value > 1000000L) {
+        return [NSString stringWithFormat:@"%1.1f MB", ((double)value / 1000000.0)];
+    } else if (value > 1000L) {
+        return [NSString stringWithFormat:@"%1.1f KB", ((double)value) / 1000.0];
+    } else if (value > 0) {
+        return [NSString stringWithFormat:@"%qu byte%@", value, (value == 0 ? @"" : @"s")];
+    }
+    return @"0";
+}
+
+@end

CacheOwnership.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -32,10 +32,17 @@
 
 
 
-@interface GoogleDriveErrorResult : NSObject {
+#import "CWLSynthesizeSingleton.h"
-    NSError *myError;
+
-}
+
-- (id)initWithAction:(NSString *)theAction data:(NSData *)theData contentType:(NSString *)theContentType httpErrorCode:(int)theHTTPStatusCode;
+@interface CacheOwnership : NSObject {
+    uid_t uid;
+    gid_t gid;
+}
+CWL_DECLARE_SINGLETON_FOR_CLASS(CacheOwnership);
+
+- (void)setUID:(uid_t)theUID gid:(gid_t)theGID;
+- (uid_t)uid;
+- (gid_t)gid;
 
-- (NSError *)error;
 @end

CacheOwnership.m

@@ -0,0 +1,59 @@
+/*
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
+ 
+ All rights reserved.
+ 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ 
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 
+ * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
+ their contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+#import "CacheOwnership.h"
+
+@implementation CacheOwnership
+CWL_SYNTHESIZE_SINGLETON_FOR_CLASS(CacheOwnership)
+
+- (id)init {
+    if (self = [super init]) {
+        uid = getuid();
+        gid = getgid();
+    }
+    return self;
+}
+- (void)setUID:(uid_t)theUID gid:(gid_t)theGID {
+    uid = theUID;
+    gid = theGID;
+}
+- (uid_t)uid {
+    return uid;
+}
+- (gid_t)gid {
+    return gid;
+}
+
+
+@end

CocoaLumberjack/CocoaLumberjack.h

@@ -0,0 +1,81 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+/**
+ * Welcome to CocoaLumberjack!
+ *
+ * The project page has a wealth of documentation if you have any questions.
+ * https://github.com/CocoaLumberjack/CocoaLumberjack
+ *
+ * If you're new to the project you may wish to read "Getting Started" at:
+ * Documentation/GettingStarted.md
+ *
+ * Otherwise, here is a quick refresher.
+ * There are three steps to using the macros:
+ *
+ * Step 1:
+ * Import the header in your implementation or prefix file:
+ *
+ * #import <CocoaLumberjack/CocoaLumberjack.h>
+ *
+ * Step 2:
+ * Define your logging level in your implementation file:
+ *
+ * // Log levels: off, error, warn, info, verbose
+ * static const DDLogLevel ddLogLevel = DDLogLevelVerbose;
+ *
+ * Step 2 [3rd party frameworks]:
+ *
+ * Define your LOG_LEVEL_DEF to a different variable/function than ddLogLevel:
+ *
+ * // #undef LOG_LEVEL_DEF // Undefine first only if needed
+ * #define LOG_LEVEL_DEF myLibLogLevel
+ *
+ * Define your logging level in your implementation file:
+ *
+ * // Log levels: off, error, warn, info, verbose
+ * static const DDLogLevel myLibLogLevel = DDLogLevelVerbose;
+ *
+ * Step 3:
+ * Replace your NSLog statements with DDLog statements according to the severity of the message.
+ *
+ * NSLog(@"Fatal error, no dohickey found!"); -> DDLogError(@"Fatal error, no dohickey found!");
+ *
+ * DDLog works exactly the same as NSLog.
+ * This means you can pass it multiple variables just like NSLog.
+ **/
+
+#import <Foundation/Foundation.h>
+
+// Disable legacy macros
+#ifndef DD_LEGACY_MACROS
+    #define DD_LEGACY_MACROS 0
+#endif
+
+// Core
+#import "DDLog.h"
+
+// Main macros
+#import "DDLogMacros.h"
+#import "DDAssertMacros.h"
+
+// Capture ASL
+#import "DDASLLogCapture.h"
+
+// Loggers
+#import "DDTTYLogger.h"
+#import "DDASLLogger.h"
+#import "DDFileLogger.h"
+

CocoaLumberjack/DDASLLogCapture.h

@@ -0,0 +1,32 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+#import "DDASLLogger.h"
+
+@protocol DDLogger;
+
+/**
+ *  This class provides the ability to capture the ASL (Apple System Logs)
+ */
+@interface DDASLLogCapture : NSObject
+
++ (void)start;
++ (void)stop;
+
+// Default log level: DDLogLevelVerbose (i.e. capture all ASL messages).
++ (DDLogLevel)captureLevel;
++ (void)setCaptureLevel:(DDLogLevel)level;
+
+@end

CocoaLumberjack/DDASLLogCapture.m

@@ -0,0 +1,230 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+#import "DDASLLogCapture.h"
+
+// Disable legacy macros
+#ifndef DD_LEGACY_MACROS
+    #define DD_LEGACY_MACROS 0
+#endif
+
+#import "DDLog.h"
+
+#include <asl.h>
+#include <notify.h>
+#include <notify_keys.h>
+#include <sys/time.h>
+
+static BOOL _cancel = YES;
+static DDLogLevel _captureLevel = DDLogLevelVerbose;
+
+#ifdef __IPHONE_8_0
+    #define DDASL_IOS_PIVOT_VERSION __IPHONE_8_0
+#endif
+#ifdef __MAC_10_10
+    #define DDASL_OSX_PIVOT_VERSION __MAC_10_10
+#endif
+
+@implementation DDASLLogCapture
+
+static aslmsg (*dd_asl_next)(aslresponse obj);
+static void (*dd_asl_release)(aslresponse obj);
+
++ (void)initialize
+{
+    #if (defined(DDASL_IOS_PIVOT_VERSION) && __IPHONE_OS_VERSION_MAX_ALLOWED >= DDASL_IOS_PIVOT_VERSION) || (defined(DDASL_OSX_PIVOT_VERSION) && __MAC_OS_X_VERSION_MAX_ALLOWED >= DDASL_OSX_PIVOT_VERSION)
+        #if __IPHONE_OS_VERSION_MIN_REQUIRED < DDASL_IOS_PIVOT_VERSION || __MAC_OS_X_VERSION_MIN_REQUIRED < DDASL_OSX_PIVOT_VERSION
+            #pragma GCC diagnostic push
+            #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+                // Building on falsely advertised SDK, targeting deprecated API
+                dd_asl_next    = &aslresponse_next;
+                dd_asl_release = &aslresponse_free;
+            #pragma GCC diagnostic pop
+        #else
+            // Building on lastest, correct SDK, targeting latest API
+            dd_asl_next    = &asl_next;
+            dd_asl_release = &asl_release;
+        #endif
+    #else
+        // Building on old SDKs, targeting deprecated API
+        dd_asl_next    = &aslresponse_next;
+        dd_asl_release = &aslresponse_free;
+    #endif
+}
+
++ (void)start {
+    // Ignore subsequent calls
+    if (!_cancel) {
+        return;
+    }
+    
+    _cancel = NO;
+    
+    dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^(void) {
+        [self captureAslLogs];
+    });
+}
+
++ (void)stop {
+    _cancel = YES;
+}
+
++ (DDLogLevel)captureLevel {
+    return _captureLevel;
+}
+
++ (void)setCaptureLevel:(DDLogLevel)level {
+    _captureLevel = level;
+}
+
+#pragma mark - Private methods
+
++ (void)configureAslQuery:(aslmsg)query {
+    const char param[] = "7";  // ASL_LEVEL_DEBUG, which is everything. We'll rely on regular DDlog log level to filter
+    
+    asl_set_query(query, ASL_KEY_LEVEL, param, ASL_QUERY_OP_LESS_EQUAL | ASL_QUERY_OP_NUMERIC);
+
+    // Don't retrieve logs from our own DDASLLogger
+    asl_set_query(query, kDDASLKeyDDLog, kDDASLDDLogValue, ASL_QUERY_OP_NOT_EQUAL);
+    
+#if !TARGET_OS_IPHONE || TARGET_IPHONE_SIMULATOR
+    int processId = [[NSProcessInfo processInfo] processIdentifier];
+    char pid[16];
+    sprintf(pid, "%d", processId);
+    asl_set_query(query, ASL_KEY_PID, pid, ASL_QUERY_OP_EQUAL | ASL_QUERY_OP_NUMERIC);
+#endif
+}
+
++ (void)aslMessageReceived:(aslmsg)msg {
+    const char* messageCString = asl_get( msg, ASL_KEY_MSG );
+    if ( messageCString == NULL )
+        return;
+
+    int flag;
+    BOOL async;
+
+    const char* levelCString = asl_get(msg, ASL_KEY_LEVEL);
+    switch (levelCString? atoi(levelCString) : 0) {
+        // By default all NSLog's with a ASL_LEVEL_WARNING level
+        case ASL_LEVEL_EMERG    :
+        case ASL_LEVEL_ALERT    :
+        case ASL_LEVEL_CRIT     : flag = DDLogFlagError;    async = NO;  break;
+        case ASL_LEVEL_ERR      : flag = DDLogFlagWarning;  async = YES; break;
+        case ASL_LEVEL_WARNING  : flag = DDLogFlagInfo;     async = YES; break;
+        case ASL_LEVEL_NOTICE   : flag = DDLogFlagDebug;    async = YES; break;
+        case ASL_LEVEL_INFO     :
+        case ASL_LEVEL_DEBUG    :
+        default                 : flag = DDLogFlagVerbose;  async = YES;  break;
+    }
+
+    if (!(_captureLevel & flag)) {
+        return;
+    }
+
+    //  NSString * sender = [NSString stringWithCString:asl_get(msg, ASL_KEY_SENDER) encoding:NSUTF8StringEncoding];
+    NSString *message = @(messageCString);
+
+    const char* secondsCString = asl_get( msg, ASL_KEY_TIME );
+    const char* nanoCString = asl_get( msg, ASL_KEY_TIME_NSEC );
+    NSTimeInterval seconds = secondsCString ? strtod(secondsCString, NULL) : [NSDate timeIntervalSinceReferenceDate] - NSTimeIntervalSince1970;
+    double nanoSeconds = nanoCString? strtod(nanoCString, NULL) : 0;
+    NSTimeInterval totalSeconds = seconds + (nanoSeconds / 1e9);
+
+    NSDate *timeStamp = [NSDate dateWithTimeIntervalSince1970:totalSeconds];
+
+    DDLogMessage *logMessage = [[DDLogMessage alloc]initWithMessage:message
+                                                              level:_captureLevel
+                                                               flag:flag
+                                                            context:0
+                                                               file:@"DDASLLogCapture"
+                                                           function:0
+                                                               line:0
+                                                                tag:nil
+                                                            options:0
+                                                          timestamp:timeStamp];
+    
+    [DDLog log:async message:logMessage];
+}
+
++ (void)captureAslLogs {
+    @autoreleasepool
+    {
+        /*
+           We use ASL_KEY_MSG_ID to see each message once, but there's no
+           obvious way to get the "next" ID. To bootstrap the process, we'll
+           search by timestamp until we've seen a message.
+         */
+
+        struct timeval timeval = {
+            .tv_sec = 0
+        };
+        gettimeofday(&timeval, NULL);
+        unsigned long long startTime = timeval.tv_sec;
+        __block unsigned long long lastSeenID = 0;
+
+        /*
+           syslogd posts kNotifyASLDBUpdate (com.apple.system.logger.message)
+           through the notify API when it saves messages to the ASL database.
+           There is some coalescing - currently it is sent at most twice per
+           second - but there is no documented guarantee about this. In any
+           case, there may be multiple messages per notification.
+
+           Notify notifications don't carry any payload, so we need to search
+           for the messages.
+         */
+        int notifyToken = 0;  // Can be used to unregister with notify_cancel().
+        notify_register_dispatch(kNotifyASLDBUpdate, &notifyToken, dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^(int token)
+        {
+            // At least one message has been posted; build a search query.
+            @autoreleasepool
+            {
+                aslmsg query = asl_new(ASL_TYPE_QUERY);
+                char stringValue[64];
+
+                if (lastSeenID > 0) {
+                    snprintf(stringValue, sizeof stringValue, "%llu", lastSeenID);
+                    asl_set_query(query, ASL_KEY_MSG_ID, stringValue, ASL_QUERY_OP_GREATER | ASL_QUERY_OP_NUMERIC);
+                } else {
+                    snprintf(stringValue, sizeof stringValue, "%llu", startTime);
+                    asl_set_query(query, ASL_KEY_TIME, stringValue, ASL_QUERY_OP_GREATER_EQUAL | ASL_QUERY_OP_NUMERIC);
+                }
+
+                [self configureAslQuery:query];
+
+                // Iterate over new messages.
+                aslmsg msg;
+                aslresponse response = asl_search(NULL, query);
+                
+                while ((msg = dd_asl_next(response)))
+                {
+                    [self aslMessageReceived:msg];
+
+                    // Keep track of which messages we've seen.
+                    lastSeenID = atoll(asl_get(msg, ASL_KEY_MSG_ID));
+                }
+                dd_asl_release(response);
+                asl_free(query);
+
+                if (_cancel) {
+                    notify_cancel(token);
+                    return;
+                }
+
+            }
+        });
+    }
+}
+
+@end

CocoaLumberjack/DDASLLogger.h

@@ -0,0 +1,54 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+#import <Foundation/Foundation.h>
+
+// Disable legacy macros
+#ifndef DD_LEGACY_MACROS
+    #define DD_LEGACY_MACROS 0
+#endif
+
+#import "DDLog.h"
+
+// Custom key set on messages sent to ASL
+extern const char* const kDDASLKeyDDLog;
+
+// Value set for kDDASLKeyDDLog
+extern const char* const kDDASLDDLogValue;
+
+/**
+ * This class provides a logger for the Apple System Log facility.
+ *
+ * As described in the "Getting Started" page,
+ * the traditional NSLog() function directs it's output to two places:
+ *
+ * - Apple System Log
+ * - StdErr (if stderr is a TTY) so log statements show up in Xcode console
+ *
+ * To duplicate NSLog() functionality you can simply add this logger and a tty logger.
+ * However, if you instead choose to use file logging (for faster performance),
+ * you may choose to use a file logger and a tty logger.
+ **/
+
+@interface DDASLLogger : DDAbstractLogger <DDLogger>
+
++ (instancetype)sharedInstance;
+
+// Inherited from DDAbstractLogger
+
+// - (id <DDLogFormatter>)logFormatter;
+// - (void)setLogFormatter:(id <DDLogFormatter>)formatter;
+
+@end

CocoaLumberjack/DDASLLogger.m

@@ -0,0 +1,121 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+#import "DDASLLogger.h"
+#import <asl.h>
+
+#if !__has_feature(objc_arc)
+#error This file must be compiled with ARC. Use -fobjc-arc flag (or convert project to ARC).
+#endif
+
+const char* const kDDASLKeyDDLog = "DDLog";
+
+const char* const kDDASLDDLogValue = "1";
+
+static DDASLLogger *sharedInstance;
+
+@interface DDASLLogger () {
+    aslclient _client;
+}
+
+@end
+
+
+@implementation DDASLLogger
+
++ (instancetype)sharedInstance {
+    static dispatch_once_t DDASLLoggerOnceToken;
+
+    dispatch_once(&DDASLLoggerOnceToken, ^{
+        sharedInstance = [[[self class] alloc] init];
+    });
+
+    return sharedInstance;
+}
+
+- (instancetype)init {
+    if (sharedInstance != nil) {
+        return nil;
+    }
+
+    if ((self = [super init])) {
+        // A default asl client is provided for the main thread,
+        // but background threads need to create their own client.
+
+        _client = asl_open(NULL, "com.apple.console", 0);
+    }
+
+    return self;
+}
+
+- (void)logMessage:(DDLogMessage *)logMessage {
+    // Skip captured log messages
+    if ([logMessage->_fileName isEqualToString:@"DDASLLogCapture"]) {
+        return;
+    }
+
+    NSString * message = _logFormatter ? [_logFormatter formatLogMessage:logMessage] : logMessage->_message;
+
+    if (logMessage) {
+        const char *msg = [message UTF8String];
+
+        size_t aslLogLevel;
+        switch (logMessage->_flag) {
+            // Note: By default ASL will filter anything above level 5 (Notice).
+            // So our mappings shouldn't go above that level.
+            case DDLogFlagError     : aslLogLevel = ASL_LEVEL_CRIT;     break;
+            case DDLogFlagWarning   : aslLogLevel = ASL_LEVEL_ERR;      break;
+            case DDLogFlagInfo      : aslLogLevel = ASL_LEVEL_WARNING;  break; // Regular NSLog's level
+            case DDLogFlagDebug     :
+            case DDLogFlagVerbose   :
+            default                 : aslLogLevel = ASL_LEVEL_NOTICE;   break;
+        }
+
+        static char const *const level_strings[] = { "0", "1", "2", "3", "4", "5", "6", "7" };
+
+        // NSLog uses the current euid to set the ASL_KEY_READ_UID.
+        uid_t const readUID = geteuid();
+
+        char readUIDString[16];
+#ifndef NS_BLOCK_ASSERTIONS
+        int l = snprintf(readUIDString, sizeof(readUIDString), "%d", readUID);
+#else
+        snprintf(readUIDString, sizeof(readUIDString), "%d", readUID);
+#endif
+
+        NSAssert(l < sizeof(readUIDString),
+                 @"Formatted euid is too long.");
+        NSAssert(aslLogLevel < (sizeof(level_strings) / sizeof(level_strings[0])),
+                 @"Unhandled ASL log level.");
+
+        aslmsg m = asl_new(ASL_TYPE_MSG);
+        if (m != NULL) {
+            if (asl_set(m, ASL_KEY_LEVEL, level_strings[aslLogLevel]) == 0 &&
+                asl_set(m, ASL_KEY_MSG, msg) == 0 &&
+                asl_set(m, ASL_KEY_READ_UID, readUIDString) == 0 &&
+                asl_set(m, kDDASLKeyDDLog, kDDASLDDLogValue) == 0) {
+                asl_send(_client, m);
+            }
+            asl_free(m);
+        }
+        //TODO handle asl_* failures non-silently?
+    }
+}
+
+- (NSString *)loggerName {
+    return @"cocoa.lumberjack.aslLogger";
+}
+
+@end

CocoaLumberjack/DDAbstractDatabaseLogger.h

@@ -0,0 +1,112 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+// Disable legacy macros
+#ifndef DD_LEGACY_MACROS
+    #define DD_LEGACY_MACROS 0
+#endif
+
+#import "DDLog.h"
+
+/**
+ * This class provides an abstract implementation of a database logger.
+ *
+ * That is, it provides the base implementation for a database logger to build atop of.
+ * All that is needed for a concrete database logger is to extend this class
+ * and override the methods in the implementation file that are prefixed with "db_".
+ **/
+
+@interface DDAbstractDatabaseLogger : DDAbstractLogger {
+    
+@protected
+    NSUInteger _saveThreshold;
+    NSTimeInterval _saveInterval;
+    NSTimeInterval _maxAge;
+    NSTimeInterval _deleteInterval;
+    BOOL _deleteOnEverySave;
+    
+    BOOL _saveTimerSuspended;
+    NSUInteger _unsavedCount;
+    dispatch_time_t _unsavedTime;
+    dispatch_source_t _saveTimer;
+    dispatch_time_t _lastDeleteTime;
+    dispatch_source_t _deleteTimer;
+}
+
+/**
+ * Specifies how often to save the data to disk.
+ * Since saving is an expensive operation (disk io) it is not done after every log statement.
+ * These properties allow you to configure how/when the logger saves to disk.
+ *
+ * A save is done when either (whichever happens first):
+ *
+ * - The number of unsaved log entries reaches saveThreshold
+ * - The amount of time since the oldest unsaved log entry was created reaches saveInterval
+ *
+ * You can optionally disable the saveThreshold by setting it to zero.
+ * If you disable the saveThreshold you are entirely dependent on the saveInterval.
+ *
+ * You can optionally disable the saveInterval by setting it to zero (or a negative value).
+ * If you disable the saveInterval you are entirely dependent on the saveThreshold.
+ *
+ * It's not wise to disable both saveThreshold and saveInterval.
+ *
+ * The default saveThreshold is 500.
+ * The default saveInterval is 60 seconds.
+ **/
+@property (assign, readwrite) NSUInteger saveThreshold;
+@property (assign, readwrite) NSTimeInterval saveInterval;
+
+/**
+ * It is likely you don't want the log entries to persist forever.
+ * Doing so would allow the database to grow infinitely large over time.
+ *
+ * The maxAge property provides a way to specify how old a log statement can get
+ * before it should get deleted from the database.
+ *
+ * The deleteInterval specifies how often to sweep for old log entries.
+ * Since deleting is an expensive operation (disk io) is is done on a fixed interval.
+ *
+ * An alternative to the deleteInterval is the deleteOnEverySave option.
+ * This specifies that old log entries should be deleted during every save operation.
+ *
+ * You can optionally disable the maxAge by setting it to zero (or a negative value).
+ * If you disable the maxAge then old log statements are not deleted.
+ *
+ * You can optionally disable the deleteInterval by setting it to zero (or a negative value).
+ *
+ * If you disable both deleteInterval and deleteOnEverySave then old log statements are not deleted.
+ *
+ * It's not wise to enable both deleteInterval and deleteOnEverySave.
+ *
+ * The default maxAge is 7 days.
+ * The default deleteInterval is 5 minutes.
+ * The default deleteOnEverySave is NO.
+ **/
+@property (assign, readwrite) NSTimeInterval maxAge;
+@property (assign, readwrite) NSTimeInterval deleteInterval;
+@property (assign, readwrite) BOOL deleteOnEverySave;
+
+/**
+ * Forces a save of any pending log entries (flushes log entries to disk).
+ **/
+- (void)savePendingLogEntries;
+
+/**
+ * Removes any log entries that are older than maxAge.
+ **/
+- (void)deleteOldLogEntries;
+
+@end

CocoaLumberjack/DDAbstractDatabaseLogger.m

@@ -0,0 +1,660 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+#import "DDAbstractDatabaseLogger.h"
+#import <math.h>
+
+
+#if !__has_feature(objc_arc)
+#error This file must be compiled with ARC. Use -fobjc-arc flag (or convert project to ARC).
+#endif
+
+@interface DDAbstractDatabaseLogger ()
+
+- (void)destroySaveTimer;
+- (void)destroyDeleteTimer;
+
+@end
+
+#pragma mark -
+
+@implementation DDAbstractDatabaseLogger
+
+- (instancetype)init {
+    if ((self = [super init])) {
+        _saveThreshold = 500;
+        _saveInterval = 60;           // 60 seconds
+        _maxAge = (60 * 60 * 24 * 7); //  7 days
+        _deleteInterval = (60 * 5);   //  5 minutes
+    }
+
+    return self;
+}
+
+- (void)dealloc {
+    [self destroySaveTimer];
+    [self destroyDeleteTimer];
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark Override Me
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+- (BOOL)db_log:(DDLogMessage *)logMessage {
+    // Override me and add your implementation.
+    //
+    // Return YES if an item was added to the buffer.
+    // Return NO if the logMessage was ignored.
+
+    return NO;
+}
+
+- (void)db_save {
+    // Override me and add your implementation.
+}
+
+- (void)db_delete {
+    // Override me and add your implementation.
+}
+
+- (void)db_saveAndDelete {
+    // Override me and add your implementation.
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark Private API
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+- (void)performSaveAndSuspendSaveTimer {
+    if (_unsavedCount > 0) {
+        if (_deleteOnEverySave) {
+            [self db_saveAndDelete];
+        } else {
+            [self db_save];
+        }
+    }
+
+    _unsavedCount = 0;
+    _unsavedTime = 0;
+
+    if (_saveTimer && !_saveTimerSuspended) {
+        dispatch_suspend(_saveTimer);
+        _saveTimerSuspended = YES;
+    }
+}
+
+- (void)performDelete {
+    if (_maxAge > 0.0) {
+        [self db_delete];
+
+        _lastDeleteTime = dispatch_time(DISPATCH_TIME_NOW, 0);
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark Timers
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+- (void)destroySaveTimer {
+    if (_saveTimer) {
+        dispatch_source_cancel(_saveTimer);
+
+        if (_saveTimerSuspended) {
+            // Must resume a timer before releasing it (or it will crash)
+            dispatch_resume(_saveTimer);
+            _saveTimerSuspended = NO;
+        }
+
+        #if !OS_OBJECT_USE_OBJC
+        dispatch_release(_saveTimer);
+        #endif
+        _saveTimer = NULL;
+    }
+}
+
+- (void)updateAndResumeSaveTimer {
+    if ((_saveTimer != NULL) && (_saveInterval > 0.0) && (_unsavedTime > 0.0)) {
+        uint64_t interval = (uint64_t)(_saveInterval * NSEC_PER_SEC);
+        dispatch_time_t startTime = dispatch_time(_unsavedTime, interval);
+
+        dispatch_source_set_timer(_saveTimer, startTime, interval, 1.0);
+
+        if (_saveTimerSuspended) {
+            dispatch_resume(_saveTimer);
+            _saveTimerSuspended = NO;
+        }
+    }
+}
+
+- (void)createSuspendedSaveTimer {
+    if ((_saveTimer == NULL) && (_saveInterval > 0.0)) {
+        _saveTimer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, self.loggerQueue);
+
+        dispatch_source_set_event_handler(_saveTimer, ^{ @autoreleasepool {
+                                                            [self performSaveAndSuspendSaveTimer];
+                                                        } });
+
+        _saveTimerSuspended = YES;
+    }
+}
+
+- (void)destroyDeleteTimer {
+    if (_deleteTimer) {
+        dispatch_source_cancel(_deleteTimer);
+        #if !OS_OBJECT_USE_OBJC
+        dispatch_release(_deleteTimer);
+        #endif
+        _deleteTimer = NULL;
+    }
+}
+
+- (void)updateDeleteTimer {
+    if ((_deleteTimer != NULL) && (_deleteInterval > 0.0) && (_maxAge > 0.0)) {
+        uint64_t interval = (uint64_t)(_deleteInterval * NSEC_PER_SEC);
+        dispatch_time_t startTime;
+
+        if (_lastDeleteTime > 0) {
+            startTime = dispatch_time(_lastDeleteTime, interval);
+        } else {
+            startTime = dispatch_time(DISPATCH_TIME_NOW, interval);
+        }
+
+        dispatch_source_set_timer(_deleteTimer, startTime, interval, 1.0);
+    }
+}
+
+- (void)createAndStartDeleteTimer {
+    if ((_deleteTimer == NULL) && (_deleteInterval > 0.0) && (_maxAge > 0.0)) {
+        _deleteTimer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, self.loggerQueue);
+
+        if (_deleteTimer != NULL) {
+            dispatch_source_set_event_handler(_deleteTimer, ^{ @autoreleasepool {
+                                                                  [self performDelete];
+                                                              } });
+
+            [self updateDeleteTimer];
+
+            if (_deleteTimer != NULL) {
+                dispatch_resume(_deleteTimer);
+            }
+        }
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark Configuration
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+- (NSUInteger)saveThreshold {
+    // The design of this method is taken from the DDAbstractLogger implementation.
+    // For extensive documentation please refer to the DDAbstractLogger implementation.
+
+    // Note: The internal implementation MUST access the colorsEnabled variable directly,
+    // This method is designed explicitly for external access.
+    //
+    // Using "self." syntax to go through this method will cause immediate deadlock.
+    // This is the intended result. Fix it by accessing the ivar directly.
+    // Great strides have been take to ensure this is safe to do. Plus it's MUCH faster.
+
+    NSAssert(![self isOnGlobalLoggingQueue], @"Core architecture requirement failure");
+    NSAssert(![self isOnInternalLoggerQueue], @"MUST access ivar directly, NOT via self.* syntax.");
+
+    dispatch_queue_t globalLoggingQueue = [DDLog loggingQueue];
+
+    __block NSUInteger result;
+
+    dispatch_sync(globalLoggingQueue, ^{
+        dispatch_sync(self.loggerQueue, ^{
+            result = _saveThreshold;
+        });
+    });
+
+    return result;
+}
+
+- (void)setSaveThreshold:(NSUInteger)threshold {
+    dispatch_block_t block = ^{
+        @autoreleasepool {
+            if (_saveThreshold != threshold) {
+                _saveThreshold = threshold;
+
+                // Since the saveThreshold has changed,
+                // we check to see if the current unsavedCount has surpassed the new threshold.
+                //
+                // If it has, we immediately save the log.
+
+                if ((_unsavedCount >= _saveThreshold) && (_saveThreshold > 0)) {
+                    [self performSaveAndSuspendSaveTimer];
+                }
+            }
+        }
+    };
+
+    // The design of the setter logic below is taken from the DDAbstractLogger implementation.
+    // For documentation please refer to the DDAbstractLogger implementation.
+
+    if ([self isOnInternalLoggerQueue]) {
+        block();
+    } else {
+        dispatch_queue_t globalLoggingQueue = [DDLog loggingQueue];
+        NSAssert(![self isOnGlobalLoggingQueue], @"Core architecture requirement failure");
+
+        dispatch_async(globalLoggingQueue, ^{
+            dispatch_async(self.loggerQueue, block);
+        });
+    }
+}
+
+- (NSTimeInterval)saveInterval {
+    // The design of this method is taken from the DDAbstractLogger implementation.
+    // For extensive documentation please refer to the DDAbstractLogger implementation.
+
+    // Note: The internal implementation MUST access the colorsEnabled variable directly,
+    // This method is designed explicitly for external access.
+    //
+    // Using "self." syntax to go through this method will cause immediate deadlock.
+    // This is the intended result. Fix it by accessing the ivar directly.
+    // Great strides have been take to ensure this is safe to do. Plus it's MUCH faster.
+
+    NSAssert(![self isOnGlobalLoggingQueue], @"Core architecture requirement failure");
+    NSAssert(![self isOnInternalLoggerQueue], @"MUST access ivar directly, NOT via self.* syntax.");
+
+    dispatch_queue_t globalLoggingQueue = [DDLog loggingQueue];
+
+    __block NSTimeInterval result;
+
+    dispatch_sync(globalLoggingQueue, ^{
+        dispatch_sync(self.loggerQueue, ^{
+            result = _saveInterval;
+        });
+    });
+
+    return result;
+}
+
+- (void)setSaveInterval:(NSTimeInterval)interval {
+    dispatch_block_t block = ^{
+        @autoreleasepool {
+            // C99 recommended floating point comparison macro
+            // Read: isLessThanOrGreaterThan(floatA, floatB)
+
+            if (/* saveInterval != interval */ islessgreater(_saveInterval, interval)) {
+                _saveInterval = interval;
+
+                // There are several cases we need to handle here.
+                //
+                // 1. If the saveInterval was previously enabled and it just got disabled,
+                //    then we need to stop the saveTimer. (And we might as well release it.)
+                //
+                // 2. If the saveInterval was previously disabled and it just got enabled,
+                //    then we need to setup the saveTimer. (Plus we might need to do an immediate save.)
+                //
+                // 3. If the saveInterval increased, then we need to reset the timer so that it fires at the later date.
+                //
+                // 4. If the saveInterval decreased, then we need to reset the timer so that it fires at an earlier date.
+                //    (Plus we might need to do an immediate save.)
+
+                if (_saveInterval > 0.0) {
+                    if (_saveTimer == NULL) {
+                        // Handles #2
+                        //
+                        // Since the saveTimer uses the unsavedTime to calculate it's first fireDate,
+                        // if a save is needed the timer will fire immediately.
+
+                        [self createSuspendedSaveTimer];
+                        [self updateAndResumeSaveTimer];
+                    } else {
+                        // Handles #3
+                        // Handles #4
+                        //
+                        // Since the saveTimer uses the unsavedTime to calculate it's first fireDate,
+                        // if a save is needed the timer will fire immediately.
+
+                        [self updateAndResumeSaveTimer];
+                    }
+                } else if (_saveTimer) {
+                    // Handles #1
+
+                    [self destroySaveTimer];
+                }
+            }
+        }
+    };
+
+    // The design of the setter logic below is taken from the DDAbstractLogger implementation.
+    // For documentation please refer to the DDAbstractLogger implementation.
+
+    if ([self isOnInternalLoggerQueue]) {
+        block();
+    } else {
+        dispatch_queue_t globalLoggingQueue = [DDLog loggingQueue];
+        NSAssert(![self isOnGlobalLoggingQueue], @"Core architecture requirement failure");
+
+        dispatch_async(globalLoggingQueue, ^{
+            dispatch_async(self.loggerQueue, block);
+        });
+    }
+}
+
+- (NSTimeInterval)maxAge {
+    // The design of this method is taken from the DDAbstractLogger implementation.
+    // For extensive documentation please refer to the DDAbstractLogger implementation.
+
+    // Note: The internal implementation MUST access the colorsEnabled variable directly,
+    // This method is designed explicitly for external access.
+    //
+    // Using "self." syntax to go through this method will cause immediate deadlock.
+    // This is the intended result. Fix it by accessing the ivar directly.
+    // Great strides have been take to ensure this is safe to do. Plus it's MUCH faster.
+
+    NSAssert(![self isOnGlobalLoggingQueue], @"Core architecture requirement failure");
+    NSAssert(![self isOnInternalLoggerQueue], @"MUST access ivar directly, NOT via self.* syntax.");
+
+    dispatch_queue_t globalLoggingQueue = [DDLog loggingQueue];
+
+    __block NSTimeInterval result;
+
+    dispatch_sync(globalLoggingQueue, ^{
+        dispatch_sync(self.loggerQueue, ^{
+            result = _maxAge;
+        });
+    });
+
+    return result;
+}
+
+- (void)setMaxAge:(NSTimeInterval)interval {
+    dispatch_block_t block = ^{
+        @autoreleasepool {
+            // C99 recommended floating point comparison macro
+            // Read: isLessThanOrGreaterThan(floatA, floatB)
+
+            if (/* maxAge != interval */ islessgreater(_maxAge, interval)) {
+                NSTimeInterval oldMaxAge = _maxAge;
+                NSTimeInterval newMaxAge = interval;
+
+                _maxAge = interval;
+
+                // There are several cases we need to handle here.
+                //
+                // 1. If the maxAge was previously enabled and it just got disabled,
+                //    then we need to stop the deleteTimer. (And we might as well release it.)
+                //
+                // 2. If the maxAge was previously disabled and it just got enabled,
+                //    then we need to setup the deleteTimer. (Plus we might need to do an immediate delete.)
+                //
+                // 3. If the maxAge was increased,
+                //    then we don't need to do anything.
+                //
+                // 4. If the maxAge was decreased,
+                //    then we should do an immediate delete.
+
+                BOOL shouldDeleteNow = NO;
+
+                if (oldMaxAge > 0.0) {
+                    if (newMaxAge <= 0.0) {
+                        // Handles #1
+
+                        [self destroyDeleteTimer];
+                    } else if (oldMaxAge > newMaxAge) {
+                        // Handles #4
+                        shouldDeleteNow = YES;
+                    }
+                } else if (newMaxAge > 0.0) {
+                    // Handles #2
+                    shouldDeleteNow = YES;
+                }
+
+                if (shouldDeleteNow) {
+                    [self performDelete];
+
+                    if (_deleteTimer) {
+                        [self updateDeleteTimer];
+                    } else {
+                        [self createAndStartDeleteTimer];
+                    }
+                }
+            }
+        }
+    };
+
+    // The design of the setter logic below is taken from the DDAbstractLogger implementation.
+    // For documentation please refer to the DDAbstractLogger implementation.
+
+    if ([self isOnInternalLoggerQueue]) {
+        block();
+    } else {
+        dispatch_queue_t globalLoggingQueue = [DDLog loggingQueue];
+        NSAssert(![self isOnGlobalLoggingQueue], @"Core architecture requirement failure");
+
+        dispatch_async(globalLoggingQueue, ^{
+            dispatch_async(self.loggerQueue, block);
+        });
+    }
+}
+
+- (NSTimeInterval)deleteInterval {
+    // The design of this method is taken from the DDAbstractLogger implementation.
+    // For extensive documentation please refer to the DDAbstractLogger implementation.
+
+    // Note: The internal implementation MUST access the colorsEnabled variable directly,
+    // This method is designed explicitly for external access.
+    //
+    // Using "self." syntax to go through this method will cause immediate deadlock.
+    // This is the intended result. Fix it by accessing the ivar directly.
+    // Great strides have been take to ensure this is safe to do. Plus it's MUCH faster.
+
+    NSAssert(![self isOnGlobalLoggingQueue], @"Core architecture requirement failure");
+    NSAssert(![self isOnInternalLoggerQueue], @"MUST access ivar directly, NOT via self.* syntax.");
+
+    dispatch_queue_t globalLoggingQueue = [DDLog loggingQueue];
+
+    __block NSTimeInterval result;
+
+    dispatch_sync(globalLoggingQueue, ^{
+        dispatch_sync(self.loggerQueue, ^{
+            result = _deleteInterval;
+        });
+    });
+
+    return result;
+}
+
+- (void)setDeleteInterval:(NSTimeInterval)interval {
+    dispatch_block_t block = ^{
+        @autoreleasepool {
+            // C99 recommended floating point comparison macro
+            // Read: isLessThanOrGreaterThan(floatA, floatB)
+
+            if (/* deleteInterval != interval */ islessgreater(_deleteInterval, interval)) {
+                _deleteInterval = interval;
+
+                // There are several cases we need to handle here.
+                //
+                // 1. If the deleteInterval was previously enabled and it just got disabled,
+                //    then we need to stop the deleteTimer. (And we might as well release it.)
+                //
+                // 2. If the deleteInterval was previously disabled and it just got enabled,
+                //    then we need to setup the deleteTimer. (Plus we might need to do an immediate delete.)
+                //
+                // 3. If the deleteInterval increased, then we need to reset the timer so that it fires at the later date.
+                //
+                // 4. If the deleteInterval decreased, then we need to reset the timer so that it fires at an earlier date.
+                //    (Plus we might need to do an immediate delete.)
+
+                if (_deleteInterval > 0.0) {
+                    if (_deleteTimer == NULL) {
+                        // Handles #2
+                        //
+                        // Since the deleteTimer uses the lastDeleteTime to calculate it's first fireDate,
+                        // if a delete is needed the timer will fire immediately.
+
+                        [self createAndStartDeleteTimer];
+                    } else {
+                        // Handles #3
+                        // Handles #4
+                        //
+                        // Since the deleteTimer uses the lastDeleteTime to calculate it's first fireDate,
+                        // if a save is needed the timer will fire immediately.
+
+                        [self updateDeleteTimer];
+                    }
+                } else if (_deleteTimer) {
+                    // Handles #1
+
+                    [self destroyDeleteTimer];
+                }
+            }
+        }
+    };
+
+    // The design of the setter logic below is taken from the DDAbstractLogger implementation.
+    // For documentation please refer to the DDAbstractLogger implementation.
+
+    if ([self isOnInternalLoggerQueue]) {
+        block();
+    } else {
+        dispatch_queue_t globalLoggingQueue = [DDLog loggingQueue];
+        NSAssert(![self isOnGlobalLoggingQueue], @"Core architecture requirement failure");
+
+        dispatch_async(globalLoggingQueue, ^{
+            dispatch_async(self.loggerQueue, block);
+        });
+    }
+}
+
+- (BOOL)deleteOnEverySave {
+    // The design of this method is taken from the DDAbstractLogger implementation.
+    // For extensive documentation please refer to the DDAbstractLogger implementation.
+
+    // Note: The internal implementation MUST access the colorsEnabled variable directly,
+    // This method is designed explicitly for external access.
+    //
+    // Using "self." syntax to go through this method will cause immediate deadlock.
+    // This is the intended result. Fix it by accessing the ivar directly.
+    // Great strides have been take to ensure this is safe to do. Plus it's MUCH faster.
+
+    NSAssert(![self isOnGlobalLoggingQueue], @"Core architecture requirement failure");
+    NSAssert(![self isOnInternalLoggerQueue], @"MUST access ivar directly, NOT via self.* syntax.");
+
+    dispatch_queue_t globalLoggingQueue = [DDLog loggingQueue];
+
+    __block BOOL result;
+
+    dispatch_sync(globalLoggingQueue, ^{
+        dispatch_sync(self.loggerQueue, ^{
+            result = _deleteOnEverySave;
+        });
+    });
+
+    return result;
+}
+
+- (void)setDeleteOnEverySave:(BOOL)flag {
+    dispatch_block_t block = ^{
+        _deleteOnEverySave = flag;
+    };
+
+    // The design of the setter logic below is taken from the DDAbstractLogger implementation.
+    // For documentation please refer to the DDAbstractLogger implementation.
+
+    if ([self isOnInternalLoggerQueue]) {
+        block();
+    } else {
+        dispatch_queue_t globalLoggingQueue = [DDLog loggingQueue];
+        NSAssert(![self isOnGlobalLoggingQueue], @"Core architecture requirement failure");
+
+        dispatch_async(globalLoggingQueue, ^{
+            dispatch_async(self.loggerQueue, block);
+        });
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark Public API
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+- (void)savePendingLogEntries {
+    dispatch_block_t block = ^{
+        @autoreleasepool {
+            [self performSaveAndSuspendSaveTimer];
+        }
+    };
+
+    if ([self isOnInternalLoggerQueue]) {
+        block();
+    } else {
+        dispatch_async(self.loggerQueue, block);
+    }
+}
+
+- (void)deleteOldLogEntries {
+    dispatch_block_t block = ^{
+        @autoreleasepool {
+            [self performDelete];
+        }
+    };
+
+    if ([self isOnInternalLoggerQueue]) {
+        block();
+    } else {
+        dispatch_async(self.loggerQueue, block);
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark DDLogger
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+- (void)didAddLogger {
+    // If you override me be sure to invoke [super didAddLogger];
+
+    [self createSuspendedSaveTimer];
+
+    [self createAndStartDeleteTimer];
+}
+
+- (void)willRemoveLogger {
+    // If you override me be sure to invoke [super willRemoveLogger];
+
+    [self performSaveAndSuspendSaveTimer];
+
+    [self destroySaveTimer];
+    [self destroyDeleteTimer];
+}
+
+- (void)logMessage:(DDLogMessage *)logMessage {
+    if ([self db_log:logMessage]) {
+        BOOL firstUnsavedEntry = (++_unsavedCount == 1);
+
+        if ((_unsavedCount >= _saveThreshold) && (_saveThreshold > 0)) {
+            [self performSaveAndSuspendSaveTimer];
+        } else if (firstUnsavedEntry) {
+            _unsavedTime = dispatch_time(DISPATCH_TIME_NOW, 0);
+            [self updateAndResumeSaveTimer];
+        }
+    }
+}
+
+- (void)flush {
+    // This method is invoked by DDLog's flushLog method.
+    //
+    // It is called automatically when the application quits,
+    // or if the developer invokes DDLog's flushLog method prior to crashing or something.
+
+    [self performSaveAndSuspendSaveTimer];
+}
+
+@end

CocoaLumberjack/DDAssertMacros.h

@@ -0,0 +1,26 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+/**
+ * NSAsset replacement that will output a log message even when assertions are disabled.
+ **/
+#define DDAssert(condition, frmt, ...)                                                \
+        if (!(condition)) {                                                           \
+            NSString *description = [NSString stringWithFormat:frmt, ## __VA_ARGS__]; \
+            DDLogError(@"%@", description);                                           \
+            NSAssert(NO, description);                                                \
+        }
+#define DDAssertCondition(condition) DDAssert(condition, @"Condition not satisfied: %s", #condition)
+

CocoaLumberjack/DDContextFilterLogFormatter.h

@@ -0,0 +1,75 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+#import <Foundation/Foundation.h>
+
+// Disable legacy macros
+#ifndef DD_LEGACY_MACROS
+    #define DD_LEGACY_MACROS 0
+#endif
+
+#import "DDLog.h"
+
+/**
+ * This class provides a log formatter that filters log statements from a logging context not on the whitelist.
+ *
+ * A log formatter can be added to any logger to format and/or filter its output.
+ * You can learn more about log formatters here:
+ * Documentation/CustomFormatters.md
+ *
+ * You can learn more about logging context's here:
+ * Documentation/CustomContext.md
+ *
+ * But here's a quick overview / refresher:
+ *
+ * Every log statement has a logging context.
+ * These come from the underlying logging macros defined in DDLog.h.
+ * The default logging context is zero.
+ * You can define multiple logging context's for use in your application.
+ * For example, logically separate parts of your app each have a different logging context.
+ * Also 3rd party frameworks that make use of Lumberjack generally use their own dedicated logging context.
+ **/
+@interface DDContextWhitelistFilterLogFormatter : NSObject <DDLogFormatter>
+
+- (instancetype)init NS_DESIGNATED_INITIALIZER;
+
+- (void)addToWhitelist:(NSUInteger)loggingContext;
+- (void)removeFromWhitelist:(NSUInteger)loggingContext;
+
+@property (readonly, copy) NSArray *whitelist;
+
+- (BOOL)isOnWhitelist:(NSUInteger)loggingContext;
+
+@end
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark -
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * This class provides a log formatter that filters log statements from a logging context on the blacklist.
+ **/
+@interface DDContextBlacklistFilterLogFormatter : NSObject <DDLogFormatter>
+
+- (instancetype)init NS_DESIGNATED_INITIALIZER;
+
+- (void)addToBlacklist:(NSUInteger)loggingContext;
+- (void)removeFromBlacklist:(NSUInteger)loggingContext;
+
+@property (readonly, copy) NSArray *blacklist;
+
+- (BOOL)isOnBlacklist:(NSUInteger)loggingContext;
+
+@end

CocoaLumberjack/DDContextFilterLogFormatter.m

@@ -0,0 +1,191 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+#import "DDContextFilterLogFormatter.h"
+#import <libkern/OSAtomic.h>
+
+#if !__has_feature(objc_arc)
+#error This file must be compiled with ARC. Use -fobjc-arc flag (or convert project to ARC).
+#endif
+
+@interface DDLoggingContextSet : NSObject
+
+- (void)addToSet:(NSUInteger)loggingContext;
+- (void)removeFromSet:(NSUInteger)loggingContext;
+
+@property (readonly, copy) NSArray *currentSet;
+
+- (BOOL)isInSet:(NSUInteger)loggingContext;
+
+@end
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark -
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+@interface DDContextWhitelistFilterLogFormatter () {
+    DDLoggingContextSet *_contextSet;
+}
+
+@end
+
+
+@implementation DDContextWhitelistFilterLogFormatter
+
+- (instancetype)init {
+    if ((self = [super init])) {
+        _contextSet = [[DDLoggingContextSet alloc] init];
+    }
+
+    return self;
+}
+
+- (void)addToWhitelist:(NSUInteger)loggingContext {
+    [_contextSet addToSet:loggingContext];
+}
+
+- (void)removeFromWhitelist:(NSUInteger)loggingContext {
+    [_contextSet removeFromSet:loggingContext];
+}
+
+- (NSArray *)whitelist {
+    return [_contextSet currentSet];
+}
+
+- (BOOL)isOnWhitelist:(NSUInteger)loggingContext {
+    return [_contextSet isInSet:loggingContext];
+}
+
+- (NSString *)formatLogMessage:(DDLogMessage *)logMessage {
+    if ([self isOnWhitelist:logMessage->_context]) {
+        return logMessage->_message;
+    } else {
+        return nil;
+    }
+}
+
+@end
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark -
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+@interface DDContextBlacklistFilterLogFormatter () {
+    DDLoggingContextSet *_contextSet;
+}
+
+@end
+
+
+@implementation DDContextBlacklistFilterLogFormatter
+
+- (instancetype)init {
+    if ((self = [super init])) {
+        _contextSet = [[DDLoggingContextSet alloc] init];
+    }
+
+    return self;
+}
+
+- (void)addToBlacklist:(NSUInteger)loggingContext {
+    [_contextSet addToSet:loggingContext];
+}
+
+- (void)removeFromBlacklist:(NSUInteger)loggingContext {
+    [_contextSet removeFromSet:loggingContext];
+}
+
+- (NSArray *)blacklist {
+    return [_contextSet currentSet];
+}
+
+- (BOOL)isOnBlacklist:(NSUInteger)loggingContext {
+    return [_contextSet isInSet:loggingContext];
+}
+
+- (NSString *)formatLogMessage:(DDLogMessage *)logMessage {
+    if ([self isOnBlacklist:logMessage->_context]) {
+        return nil;
+    } else {
+        return logMessage->_message;
+    }
+}
+
+@end
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark -
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+@interface DDLoggingContextSet () {
+    OSSpinLock _lock;
+    NSMutableSet *_set;
+}
+
+@end
+
+
+@implementation DDLoggingContextSet
+
+- (instancetype)init {
+    if ((self = [super init])) {
+        _set = [[NSMutableSet alloc] init];
+    }
+
+    return self;
+}
+
+- (void)addToSet:(NSUInteger)loggingContext {
+    OSSpinLockLock(&_lock);
+    {
+        [_set addObject:@(loggingContext)];
+    }
+    OSSpinLockUnlock(&_lock);
+}
+
+- (void)removeFromSet:(NSUInteger)loggingContext {
+    OSSpinLockLock(&_lock);
+    {
+        [_set removeObject:@(loggingContext)];
+    }
+    OSSpinLockUnlock(&_lock);
+}
+
+- (NSArray *)currentSet {
+    NSArray *result = nil;
+
+    OSSpinLockLock(&_lock);
+    {
+        result = [_set allObjects];
+    }
+    OSSpinLockUnlock(&_lock);
+
+    return result;
+}
+
+- (BOOL)isInSet:(NSUInteger)loggingContext {
+    BOOL result = NO;
+
+    OSSpinLockLock(&_lock);
+    {
+        result = [_set containsObject:@(loggingContext)];
+    }
+    OSSpinLockUnlock(&_lock);
+
+    return result;
+}
+
+@end

CocoaLumberjack/DDDispatchQueueLogFormatter.h

@@ -0,0 +1,135 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+#import <Foundation/Foundation.h>
+#import <libkern/OSAtomic.h>
+
+// Disable legacy macros
+#ifndef DD_LEGACY_MACROS
+    #define DD_LEGACY_MACROS 0
+#endif
+
+#import "DDLog.h"
+
+
+/**
+ * This class provides a log formatter that prints the dispatch_queue label instead of the mach_thread_id.
+ *
+ * A log formatter can be added to any logger to format and/or filter its output.
+ * You can learn more about log formatters here:
+ * Documentation/CustomFormatters.md
+ *
+ * A typical NSLog (or DDTTYLogger) prints detailed info as [<process_id>:<thread_id>].
+ * For example:
+ *
+ * 2011-10-17 20:21:45.435 AppName[19928:5207] Your log message here
+ *
+ * Where:
+ * - 19928 = process id
+ * -  5207 = thread id (mach_thread_id printed in hex)
+ *
+ * When using grand central dispatch (GCD), this information is less useful.
+ * This is because a single serial dispatch queue may be run on any thread from an internally managed thread pool.
+ * For example:
+ *
+ * 2011-10-17 20:32:31.111 AppName[19954:4d07] Message from my_serial_dispatch_queue
+ * 2011-10-17 20:32:31.112 AppName[19954:5207] Message from my_serial_dispatch_queue
+ * 2011-10-17 20:32:31.113 AppName[19954:2c55] Message from my_serial_dispatch_queue
+ *
+ * This formatter allows you to replace the standard [box:info] with the dispatch_queue name.
+ * For example:
+ *
+ * 2011-10-17 20:32:31.111 AppName[img-scaling] Message from my_serial_dispatch_queue
+ * 2011-10-17 20:32:31.112 AppName[img-scaling] Message from my_serial_dispatch_queue
+ * 2011-10-17 20:32:31.113 AppName[img-scaling] Message from my_serial_dispatch_queue
+ *
+ * If the dispatch_queue doesn't have a set name, then it falls back to the thread name.
+ * If the current thread doesn't have a set name, then it falls back to the mach_thread_id in hex (like normal).
+ *
+ * Note: If manually creating your own background threads (via NSThread/alloc/init or NSThread/detachNeThread),
+ * you can use [[NSThread currentThread] setName:(NSString *)].
+ **/
+@interface DDDispatchQueueLogFormatter : NSObject <DDLogFormatter>
+
+/**
+ * Standard init method.
+ * Configure using properties as desired.
+ **/
+- (instancetype)init NS_DESIGNATED_INITIALIZER;
+
+/**
+ * The minQueueLength restricts the minimum size of the [detail box].
+ * If the minQueueLength is set to 0, there is no restriction.
+ *
+ * For example, say a dispatch_queue has a label of "diskIO":
+ *
+ * If the minQueueLength is 0: [diskIO]
+ * If the minQueueLength is 4: [diskIO]
+ * If the minQueueLength is 5: [diskIO]
+ * If the minQueueLength is 6: [diskIO]
+ * If the minQueueLength is 7: [diskIO ]
+ * If the minQueueLength is 8: [diskIO  ]
+ *
+ * The default minQueueLength is 0 (no minimum, so [detail box] won't be padded).
+ *
+ * If you want every [detail box] to have the exact same width,
+ * set both minQueueLength and maxQueueLength to the same value.
+ **/
+@property (assign, atomic) NSUInteger minQueueLength;
+
+/**
+ * The maxQueueLength restricts the number of characters that will be inside the [detail box].
+ * If the maxQueueLength is 0, there is no restriction.
+ *
+ * For example, say a dispatch_queue has a label of "diskIO":
+ *
+ * If the maxQueueLength is 0: [diskIO]
+ * If the maxQueueLength is 4: [disk]
+ * If the maxQueueLength is 5: [diskI]
+ * If the maxQueueLength is 6: [diskIO]
+ * If the maxQueueLength is 7: [diskIO]
+ * If the maxQueueLength is 8: [diskIO]
+ *
+ * The default maxQueueLength is 0 (no maximum, so [detail box] won't be truncated).
+ *
+ * If you want every [detail box] to have the exact same width,
+ * set both minQueueLength and maxQueueLength to the same value.
+ **/
+@property (assign, atomic) NSUInteger maxQueueLength;
+
+/**
+ * Sometimes queue labels have long names like "com.apple.main-queue",
+ * but you'd prefer something shorter like simply "main".
+ *
+ * This method allows you to set such preferred replacements.
+ * The above example is set by default.
+ *
+ * To remove/undo a previous replacement, invoke this method with nil for the 'shortLabel' parameter.
+ **/
+- (NSString *)replacementStringForQueueLabel:(NSString *)longLabel;
+- (void)setReplacementString:(NSString *)shortLabel forQueueLabel:(NSString *)longLabel;
+
+@end
+
+/**
+ * Method declarations that make it easier to extend/modify DDDispatchQueueLogFormatter
+ **/
+@interface DDDispatchQueueLogFormatter (OverridableMethods)
+
+- (NSString *)stringFromDate:(NSDate *)date;
+- (NSString *)queueThreadLabelForLogMessage:(DDLogMessage *)logMessage;
+- (NSString *)formatLogMessage:(DDLogMessage *)logMessage;
+
+@end

CocoaLumberjack/DDDispatchQueueLogFormatter.m

@@ -0,0 +1,247 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+#import "DDDispatchQueueLogFormatter.h"
+#import <libkern/OSAtomic.h>
+
+
+#if !__has_feature(objc_arc)
+#error This file must be compiled with ARC. Use -fobjc-arc flag (or convert project to ARC).
+#endif
+
+@interface DDDispatchQueueLogFormatter () {
+    NSString *_dateFormatString;
+    
+    int32_t _atomicLoggerCount;
+    NSDateFormatter *_threadUnsafeDateFormatter; // Use [self stringFromDate]
+    
+    OSSpinLock _lock;
+    
+    NSUInteger _minQueueLength;           // _prefix == Only access via atomic property
+    NSUInteger _maxQueueLength;           // _prefix == Only access via atomic property
+    NSMutableDictionary *_replacements;   // _prefix == Only access from within spinlock
+}
+
+@end
+
+
+@implementation DDDispatchQueueLogFormatter
+
+- (instancetype)init {
+    if ((self = [super init])) {
+        _dateFormatString = @"yyyy-MM-dd HH:mm:ss:SSS";
+
+        _atomicLoggerCount = 0;
+        _threadUnsafeDateFormatter = nil;
+
+        _minQueueLength = 0;
+        _maxQueueLength = 0;
+        _replacements = [[NSMutableDictionary alloc] init];
+
+        // Set default replacements:
+
+        _replacements[@"com.apple.main-thread"] = @"main";
+    }
+
+    return self;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark Configuration
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+@synthesize minQueueLength = _minQueueLength;
+@synthesize maxQueueLength = _maxQueueLength;
+
+- (NSString *)replacementStringForQueueLabel:(NSString *)longLabel {
+    NSString *result = nil;
+
+    OSSpinLockLock(&_lock);
+    {
+        result = _replacements[longLabel];
+    }
+    OSSpinLockUnlock(&_lock);
+
+    return result;
+}
+
+- (void)setReplacementString:(NSString *)shortLabel forQueueLabel:(NSString *)longLabel {
+    OSSpinLockLock(&_lock);
+    {
+        if (shortLabel) {
+            _replacements[longLabel] = shortLabel;
+        } else {
+            [_replacements removeObjectForKey:longLabel];
+        }
+    }
+    OSSpinLockUnlock(&_lock);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark DDLogFormatter
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+- (NSString *)stringFromDate:(NSDate *)date {
+    int32_t loggerCount = OSAtomicAdd32(0, &_atomicLoggerCount);
+
+    NSString *calendarIdentifier = nil;
+
+#if defined(__IPHONE_8_0) || defined(__MAC_10_10)
+    calendarIdentifier = NSCalendarIdentifierGregorian;
+#else
+    calendarIdentifier = NSGregorianCalendar;
+#endif
+
+    if (loggerCount <= 1) {
+        // Single-threaded mode.
+
+        if (_threadUnsafeDateFormatter == nil) {
+            _threadUnsafeDateFormatter = [[NSDateFormatter alloc] init];
+            [_threadUnsafeDateFormatter setFormatterBehavior:NSDateFormatterBehavior10_4];
+            [_threadUnsafeDateFormatter setDateFormat:_dateFormatString];
+        }
+
+        [_threadUnsafeDateFormatter setCalendar:[[NSCalendar alloc] initWithCalendarIdentifier:calendarIdentifier]];
+        return [_threadUnsafeDateFormatter stringFromDate:date];
+    } else {
+        // Multi-threaded mode.
+        // NSDateFormatter is NOT thread-safe.
+
+        NSString *key = @"DispatchQueueLogFormatter_NSDateFormatter";
+
+        NSMutableDictionary *threadDictionary = [[NSThread currentThread] threadDictionary];
+        NSDateFormatter *dateFormatter = threadDictionary[key];
+
+        if (dateFormatter == nil) {
+            dateFormatter = [[NSDateFormatter alloc] init];
+            [dateFormatter setFormatterBehavior:NSDateFormatterBehavior10_4];
+            [dateFormatter setDateFormat:_dateFormatString];
+
+            threadDictionary[key] = dateFormatter;
+        }
+
+        [dateFormatter setCalendar:[[NSCalendar alloc] initWithCalendarIdentifier:calendarIdentifier]];
+        return [dateFormatter stringFromDate:date];
+    }
+}
+
+- (NSString *)queueThreadLabelForLogMessage:(DDLogMessage *)logMessage {
+    // As per the DDLogFormatter contract, this method is always invoked on the same thread/dispatch_queue
+
+    NSUInteger minQueueLength = self.minQueueLength;
+    NSUInteger maxQueueLength = self.maxQueueLength;
+
+    // Get the name of the queue, thread, or machID (whichever we are to use).
+
+    NSString *queueThreadLabel = nil;
+
+    BOOL useQueueLabel = YES;
+    BOOL useThreadName = NO;
+
+    if (logMessage->_queueLabel) {
+        // If you manually create a thread, it's dispatch_queue will have one of the thread names below.
+        // Since all such threads have the same name, we'd prefer to use the threadName or the machThreadID.
+
+        NSArray *names = @[
+            @"com.apple.root.low-priority",
+            @"com.apple.root.default-priority",
+            @"com.apple.root.high-priority",
+            @"com.apple.root.low-overcommit-priority",
+            @"com.apple.root.default-overcommit-priority",
+            @"com.apple.root.high-overcommit-priority"
+        ];
+
+        for (NSString * name in names) {
+            if ([logMessage->_queueLabel isEqualToString:name]) {
+                useQueueLabel = NO;
+                useThreadName = [logMessage->_threadName length] > 0;
+                break;
+            }
+        }
+    } else {
+        useQueueLabel = NO;
+        useThreadName = [logMessage->_threadName length] > 0;
+    }
+
+    if (useQueueLabel || useThreadName) {
+        NSString *fullLabel;
+        NSString *abrvLabel;
+
+        if (useQueueLabel) {
+            fullLabel = logMessage->_queueLabel;
+        } else {
+            fullLabel = logMessage->_threadName;
+        }
+
+        OSSpinLockLock(&_lock);
+        {
+            abrvLabel = _replacements[fullLabel];
+        }
+        OSSpinLockUnlock(&_lock);
+
+        if (abrvLabel) {
+            queueThreadLabel = abrvLabel;
+        } else {
+            queueThreadLabel = fullLabel;
+        }
+    } else {
+        queueThreadLabel = logMessage->_threadID;
+    }
+
+    // Now use the thread label in the output
+
+    NSUInteger labelLength = [queueThreadLabel length];
+
+    // labelLength > maxQueueLength : truncate
+    // labelLength < minQueueLength : padding
+    //                              : exact
+
+    if ((maxQueueLength > 0) && (labelLength > maxQueueLength)) {
+        // Truncate
+
+        return [queueThreadLabel substringToIndex:maxQueueLength];
+    } else if (labelLength < minQueueLength) {
+        // Padding
+
+        NSUInteger numSpaces = minQueueLength - labelLength;
+
+        char spaces[numSpaces + 1];
+        memset(spaces, ' ', numSpaces);
+        spaces[numSpaces] = '\0';
+
+        return [NSString stringWithFormat:@"%@%s", queueThreadLabel, spaces];
+    } else {
+        // Exact
+
+        return queueThreadLabel;
+    }
+}
+
+- (NSString *)formatLogMessage:(DDLogMessage *)logMessage {
+    NSString *timestamp = [self stringFromDate:(logMessage->_timestamp)];
+    NSString *queueThreadLabel = [self queueThreadLabelForLogMessage:logMessage];
+
+    return [NSString stringWithFormat:@"%@ [%@] %@", timestamp, queueThreadLabel, logMessage->_message];
+}
+
+- (void)didAddToLogger:(id <DDLogger>  __attribute__((unused)))logger {
+    OSAtomicIncrement32(&_atomicLoggerCount);
+}
+
+- (void)willRemoveFromLogger:(id <DDLogger> __attribute__((unused)))logger {
+    OSAtomicDecrement32(&_atomicLoggerCount);
+}
+
+@end

CocoaLumberjack/DDFileLogger.h

@@ -0,0 +1,391 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+// Disable legacy macros
+#ifndef DD_LEGACY_MACROS
+    #define DD_LEGACY_MACROS 0
+#endif
+
+#import "DDLog.h"
+
+@class DDLogFileInfo;
+
+/**
+ * This class provides a logger to write log statements to a file.
+ **/
+
+
+// Default configuration and safety/sanity values.
+//
+// maximumFileSize         -> kDDDefaultLogMaxFileSize
+// rollingFrequency        -> kDDDefaultLogRollingFrequency
+// maximumNumberOfLogFiles -> kDDDefaultLogMaxNumLogFiles
+// logFilesDiskQuota       -> kDDDefaultLogFilesDiskQuota
+//
+// You should carefully consider the proper configuration values for your application.
+
+extern unsigned long long const kDDDefaultLogMaxFileSize;
+extern NSTimeInterval     const kDDDefaultLogRollingFrequency;
+extern NSUInteger         const kDDDefaultLogMaxNumLogFiles;
+extern unsigned long long const kDDDefaultLogFilesDiskQuota;
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark -
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+// The LogFileManager protocol is designed to allow you to control all aspects of your log files.
+//
+// The primary purpose of this is to allow you to do something with the log files after they have been rolled.
+// Perhaps you want to compress them to save disk space.
+// Perhaps you want to upload them to an FTP server.
+// Perhaps you want to run some analytics on the file.
+//
+// A default LogFileManager is, of course, provided.
+// The default LogFileManager simply deletes old log files according to the maximumNumberOfLogFiles property.
+//
+// This protocol provides various methods to fetch the list of log files.
+//
+// There are two variants: sorted and unsorted.
+// If sorting is not necessary, the unsorted variant is obviously faster.
+// The sorted variant will return an array sorted by when the log files were created,
+// with the most recently created log file at index 0, and the oldest log file at the end of the array.
+//
+// You can fetch only the log file paths (full path including name), log file names (name only),
+// or an array of DDLogFileInfo objects.
+// The DDLogFileInfo class is documented below, and provides a handy wrapper that
+// gives you easy access to various file attributes such as the creation date or the file size.
+
+@protocol DDLogFileManager <NSObject>
+@required
+
+// Public properties
+
+/**
+ * The maximum number of archived log files to keep on disk.
+ * For example, if this property is set to 3,
+ * then the LogFileManager will only keep 3 archived log files (plus the current active log file) on disk.
+ * Once the active log file is rolled/archived, then the oldest of the existing 3 rolled/archived log files is deleted.
+ *
+ * You may optionally disable this option by setting it to zero.
+ **/
+@property (readwrite, assign, atomic) NSUInteger maximumNumberOfLogFiles;
+
+/**
+ * The maximum space that logs can take. On rolling logfile all old logfiles that exceed logFilesDiskQuota will
+ * be deleted.
+ *
+ * You may optionally disable this option by setting it to zero.
+ **/
+@property (readwrite, assign, atomic) unsigned long long logFilesDiskQuota;
+
+// Public methods
+
+- (NSString *)logsDirectory;
+
+- (NSArray *)unsortedLogFilePaths;
+- (NSArray *)unsortedLogFileNames;
+- (NSArray *)unsortedLogFileInfos;
+
+- (NSArray *)sortedLogFilePaths;
+- (NSArray *)sortedLogFileNames;
+- (NSArray *)sortedLogFileInfos;
+
+// Private methods (only to be used by DDFileLogger)
+
+- (NSString *)createNewLogFile;
+
+@optional
+
+// Notifications from DDFileLogger
+
+- (void)didArchiveLogFile:(NSString *)logFilePath;
+- (void)didRollAndArchiveLogFile:(NSString *)logFilePath;
+
+@end
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark -
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Default log file manager.
+ *
+ * All log files are placed inside the logsDirectory.
+ * If a specific logsDirectory isn't specified, the default directory is used.
+ * On Mac, this is in ~/Library/Logs/<Application Name>.
+ * On iPhone, this is in ~/Library/Caches/Logs.
+ *
+ * Log files are named "<bundle identifier> <date> <time>.log"
+ * Example: com.organization.myapp 2013-12-03 17-14.log
+ *
+ * Archived log files are automatically deleted according to the maximumNumberOfLogFiles property.
+ **/
+@interface DDLogFileManagerDefault : NSObject <DDLogFileManager>
+
+- (instancetype)init;
+- (instancetype)initWithLogsDirectory:(NSString *)logsDirectory NS_DESIGNATED_INITIALIZER;
+#if TARGET_OS_IPHONE
+/*
+ * Calling this constructor you can override the default "automagically" chosen NSFileProtection level.
+ * Useful if you are writing a command line utility / CydiaSubstrate addon for iOS that has no NSBundle
+ * or like SpringBoard no BackgroundModes key in the NSBundle:
+ *    iPhone:~ root# cycript -p SpringBoard
+ *    cy# [NSBundle mainBundle]
+ *    #"NSBundle </System/Library/CoreServices/SpringBoard.app> (loaded)"
+ *    cy# [[NSBundle mainBundle] objectForInfoDictionaryKey:@"UIBackgroundModes"];
+ *    null
+ *    cy#
+ **/
+- (instancetype)initWithLogsDirectory:(NSString *)logsDirectory defaultFileProtectionLevel:(NSString *)fileProtectionLevel;
+#endif
+
+/*
+ * Methods to override.
+ *
+ * Log files are named "<bundle identifier> <date> <time>.log"
+ * Example: com.organization.myapp 2013-12-03 17-14.log
+ *
+ * If you wish to change default filename, you can override following two methods.
+ * - newLogFileName method would be called on new logfile creation.
+ * - isLogFile: method would be called to filter logfiles from all other files in logsDirectory.
+ *   You have to parse given filename and return YES if it is logFile.
+ *
+ * **NOTE**
+ * newLogFileName returns filename. If appropriate file already exists, number would be added
+ * to filename before extension. You have to handle this case in isLogFile: method.
+ *
+ * Example:
+ * - newLogFileName returns "com.organization.myapp 2013-12-03.log",
+ *   file "com.organization.myapp 2013-12-03.log" would be created.
+ * - after some time "com.organization.myapp 2013-12-03.log" is archived
+ * - newLogFileName again returns "com.organization.myapp 2013-12-03.log",
+ *   file "com.organization.myapp 2013-12-03 2.log" would be created.
+ * - after some time "com.organization.myapp 2013-12-03 1.log" is archived
+ * - newLogFileName again returns "com.organization.myapp 2013-12-03.log",
+ *   file "com.organization.myapp 2013-12-03 3.log" would be created.
+ **/
+@property (readonly, copy) NSString *newLogFileName;
+- (BOOL)isLogFile:(NSString *)fileName;
+
+/* Inherited from DDLogFileManager protocol:
+
+   @property (readwrite, assign, atomic) NSUInteger maximumNumberOfLogFiles;
+   @property (readwrite, assign, atomic) NSUInteger logFilesDiskQuota;
+
+   - (NSString *)logsDirectory;
+
+   - (NSArray *)unsortedLogFilePaths;
+   - (NSArray *)unsortedLogFileNames;
+   - (NSArray *)unsortedLogFileInfos;
+
+   - (NSArray *)sortedLogFilePaths;
+   - (NSArray *)sortedLogFileNames;
+   - (NSArray *)sortedLogFileInfos;
+
+ */
+
+@end
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark -
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Most users will want file log messages to be prepended with the date and time.
+ * Rather than forcing the majority of users to write their own formatter,
+ * we will supply a logical default formatter.
+ * Users can easily replace this formatter with their own by invoking the setLogFormatter method.
+ * It can also be removed by calling setLogFormatter, and passing a nil parameter.
+ *
+ * In addition to the convenience of having a logical default formatter,
+ * it will also provide a template that makes it easy for developers to copy and change.
+ **/
+@interface DDLogFileFormatterDefault : NSObject <DDLogFormatter>
+
+- (instancetype)init;
+- (instancetype)initWithDateFormatter:(NSDateFormatter *)dateFormatter NS_DESIGNATED_INITIALIZER;
+
+@end
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark -
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+@interface DDFileLogger : DDAbstractLogger <DDLogger>
+
+- (instancetype)init;
+- (instancetype)initWithLogFileManager:(id <DDLogFileManager>)logFileManager NS_DESIGNATED_INITIALIZER;
+
+/**
+ * Log File Rolling:
+ *
+ * maximumFileSize:
+ *   The approximate maximum size to allow log files to grow.
+ *   If a log file is larger than this value after a log statement is appended,
+ *   then the log file is rolled.
+ *
+ * rollingFrequency
+ *   How often to roll the log file.
+ *   The frequency is given as an NSTimeInterval, which is a double that specifies the interval in seconds.
+ *   Once the log file gets to be this old, it is rolled.
+ *
+ * Both the maximumFileSize and the rollingFrequency are used to manage rolling.
+ * Whichever occurs first will cause the log file to be rolled.
+ *
+ * For example:
+ * The rollingFrequency is 24 hours,
+ * but the log file surpasses the maximumFileSize after only 20 hours.
+ * The log file will be rolled at that 20 hour mark.
+ * A new log file will be created, and the 24 hour timer will be restarted.
+ *
+ * You may optionally disable rolling due to filesize by setting maximumFileSize to zero.
+ * If you do so, rolling is based solely on rollingFrequency.
+ *
+ * You may optionally disable rolling due to time by setting rollingFrequency to zero (or any non-positive number).
+ * If you do so, rolling is based solely on maximumFileSize.
+ *
+ * If you disable both maximumFileSize and rollingFrequency, then the log file won't ever be rolled.
+ * This is strongly discouraged.
+ **/
+@property (readwrite, assign) unsigned long long maximumFileSize;
+@property (readwrite, assign) NSTimeInterval rollingFrequency;
+@property (readwrite, assign, atomic) BOOL doNotReuseLogFiles;
+
+/**
+ * The DDLogFileManager instance can be used to retrieve the list of log files,
+ * and configure the maximum number of archived log files to keep.
+ *
+ * @see DDLogFileManager.maximumNumberOfLogFiles
+ **/
+@property (strong, nonatomic, readonly) id <DDLogFileManager> logFileManager;
+
+/**
+ * When using a custom formatter you can set the logMessage method not to append
+ * '\n' character after each output. This allows for some greater flexibility with
+ * custom formatters. Default value is YES.
+ **/
+
+@property (nonatomic, readwrite, assign) BOOL automaticallyAppendNewlineForCustomFormatters;
+
+// You can optionally force the current log file to be rolled with this method.
+// CompletionBlock will be called on main queue.
+
+- (void)rollLogFileWithCompletionBlock:(void (^)())completionBlock;
+
+// Method is deprecated. Use rollLogFileWithCompletionBlock: method instead.
+
+- (void)rollLogFile __attribute((deprecated));
+
+// Inherited from DDAbstractLogger
+
+// - (id <DDLogFormatter>)logFormatter;
+// - (void)setLogFormatter:(id <DDLogFormatter>)formatter;
+
+/**
+ * Returns the log file that should be used.
+ * If there is an existing log file that is suitable,
+ * within the constraints of maximumFileSize and rollingFrequency, then it is returned.
+ *
+ * Otherwise a new file is created and returned.
+ **/
+- (DDLogFileInfo *)currentLogFileInfo;
+
+@end
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark -
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * DDLogFileInfo is a simple class that provides access to various file attributes.
+ * It provides good performance as it only fetches the information if requested,
+ * and it caches the information to prevent duplicate fetches.
+ *
+ * It was designed to provide quick snapshots of the current state of log files,
+ * and to help sort log files in an array.
+ *
+ * This class does not monitor the files, or update it's cached attribute values if the file changes on disk.
+ * This is not what the class was designed for.
+ *
+ * If you absolutely must get updated values,
+ * you can invoke the reset method which will clear the cache.
+ **/
+@interface DDLogFileInfo : NSObject
+
+@property (strong, nonatomic, readonly) NSString *filePath;
+@property (strong, nonatomic, readonly) NSString *fileName;
+
+@property (strong, nonatomic, readonly) NSDictionary *fileAttributes;
+
+@property (strong, nonatomic, readonly) NSDate *creationDate;
+@property (strong, nonatomic, readonly) NSDate *modificationDate;
+
+@property (nonatomic, readonly) unsigned long long fileSize;
+
+@property (nonatomic, readonly) NSTimeInterval age;
+
+@property (nonatomic, readwrite) BOOL isArchived;
+
++ (instancetype)logFileWithPath:(NSString *)filePath;
+
+- (instancetype)initWithFilePath:(NSString *)filePath NS_DESIGNATED_INITIALIZER;
+
+- (void)reset;
+- (void)renameFile:(NSString *)newFileName;
+
+#if TARGET_IPHONE_SIMULATOR
+
+// So here's the situation.
+// Extended attributes are perfect for what we're trying to do here (marking files as archived).
+// This is exactly what extended attributes were designed for.
+//
+// But Apple screws us over on the simulator.
+// Everytime you build-and-go, they copy the application into a new folder on the hard drive,
+// and as part of the process they strip extended attributes from our log files.
+// Normally, a copy of a file preserves extended attributes.
+// So obviously Apple has gone to great lengths to piss us off.
+//
+// Thus we use a slightly different tactic for marking log files as archived in the simulator.
+// That way it "just works" and there's no confusion when testing.
+//
+// The difference in method names is indicative of the difference in functionality.
+// On the simulator we add an attribute by appending a filename extension.
+//
+// For example:
+// "mylog.txt" -> "mylog.archived.txt"
+// "mylog"     -> "mylog.archived"
+
+- (BOOL)hasExtensionAttributeWithName:(NSString *)attrName;
+
+- (void)addExtensionAttributeWithName:(NSString *)attrName;
+- (void)removeExtensionAttributeWithName:(NSString *)attrName;
+
+#else /* if TARGET_IPHONE_SIMULATOR */
+
+// Normal use of extended attributes used everywhere else,
+// such as on Macs and on iPhone devices.
+
+- (BOOL)hasExtendedAttributeWithName:(NSString *)attrName;
+
+- (void)addExtendedAttributeWithName:(NSString *)attrName;
+- (void)removeExtendedAttributeWithName:(NSString *)attrName;
+
+#endif /* if TARGET_IPHONE_SIMULATOR */
+
+- (NSComparisonResult)reverseCompareByCreationDate:(DDLogFileInfo *)another;
+- (NSComparisonResult)reverseCompareByModificationDate:(DDLogFileInfo *)another;
+
+@end

CocoaLumberjack/DDLegacyMacros.h

@@ -0,0 +1,75 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+/**
+ * Legacy macros used for 1.9.x backwards compatibility.
+ *
+ * Imported by default when importing a DDLog.h directly and DD_LEGACY_MACROS is not defined and set to 0.
+ **/
+#if DD_LEGACY_MACROS
+
+#warning CocoaLumberjack 1.9.x legacy macros enabled. \
+Disable legacy macros by importing CocoaLumberjack.h or DDLogMacros.h instead of DDLog.h or add `#define DD_LEGACY_MACROS 0` before importing DDLog.h.
+
+#ifndef LOG_LEVEL_DEF
+    #define LOG_LEVEL_DEF ddLogLevel
+#endif
+
+#define LOG_FLAG_ERROR    DDLogFlagError
+#define LOG_FLAG_WARN     DDLogFlagWarning
+#define LOG_FLAG_INFO     DDLogFlagInfo
+#define LOG_FLAG_DEBUG    DDLogFlagDebug
+#define LOG_FLAG_VERBOSE  DDLogFlagVerbose
+
+#define LOG_LEVEL_OFF     DDLogLevelOff
+#define LOG_LEVEL_ERROR   DDLogLevelError
+#define LOG_LEVEL_WARN    DDLogLevelWarning
+#define LOG_LEVEL_INFO    DDLogLevelInfo
+#define LOG_LEVEL_DEBUG   DDLogLevelDebug
+#define LOG_LEVEL_VERBOSE DDLogLevelVerbose
+#define LOG_LEVEL_ALL     DDLogLevelAll
+
+#define LOG_ASYNC_ENABLED YES
+
+#define LOG_ASYNC_ERROR    ( NO && LOG_ASYNC_ENABLED)
+#define LOG_ASYNC_WARN     (YES && LOG_ASYNC_ENABLED)
+#define LOG_ASYNC_INFO     (YES && LOG_ASYNC_ENABLED)
+#define LOG_ASYNC_DEBUG    (YES && LOG_ASYNC_ENABLED)
+#define LOG_ASYNC_VERBOSE  (YES && LOG_ASYNC_ENABLED)
+
+#define LOG_MACRO(isAsynchronous, lvl, flg, ctx, atag, fnct, frmt, ...) \
+        [DDLog log : isAsynchronous                                     \
+             level : lvl                                                \
+              flag : flg                                                \
+           context : ctx                                                \
+              file : __FILE__                                           \
+          function : fnct                                               \
+              line : __LINE__                                           \
+               tag : atag                                               \
+            format : (frmt), ## __VA_ARGS__]
+
+#define LOG_MAYBE(async, lvl, flg, ctx, fnct, frmt, ...)                       \
+        do { if(lvl & flg) LOG_MACRO(async, lvl, flg, ctx, nil, fnct, frmt, ##__VA_ARGS__); } while(0)
+
+#define LOG_OBJC_MAYBE(async, lvl, flg, ctx, frmt, ...) \
+        LOG_MAYBE(async, lvl, flg, ctx, __PRETTY_FUNCTION__, frmt, ## __VA_ARGS__)
+
+#define DDLogError(frmt, ...)   LOG_OBJC_MAYBE(LOG_ASYNC_ERROR,   LOG_LEVEL_DEF, LOG_FLAG_ERROR,   0, frmt, ##__VA_ARGS__)
+#define DDLogWarn(frmt, ...)    LOG_OBJC_MAYBE(LOG_ASYNC_WARN,    LOG_LEVEL_DEF, LOG_FLAG_WARN,    0, frmt, ##__VA_ARGS__)
+#define DDLogInfo(frmt, ...)    LOG_OBJC_MAYBE(LOG_ASYNC_INFO,    LOG_LEVEL_DEF, LOG_FLAG_INFO,    0, frmt, ##__VA_ARGS__)
+#define DDLogDebug(frmt, ...)   LOG_OBJC_MAYBE(LOG_ASYNC_DEBUG,   LOG_LEVEL_DEF, LOG_FLAG_DEBUG,   0, frmt, ##__VA_ARGS__)
+#define DDLogVerbose(frmt, ...) LOG_OBJC_MAYBE(LOG_ASYNC_VERBOSE, LOG_LEVEL_DEF, LOG_FLAG_VERBOSE, 0, frmt, ##__VA_ARGS__)
+
+#endif

CocoaLumberjack/DDLog+LOGV.h

@@ -0,0 +1,83 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+// Disable legacy macros
+#ifndef DD_LEGACY_MACROS
+    #define DD_LEGACY_MACROS 0
+#endif
+
+#import "DDLog.h"
+
+/**
+ * The constant/variable/method responsible for controlling the current log level.
+ **/
+#ifndef LOG_LEVEL_DEF
+    #define LOG_LEVEL_DEF ddLogLevel
+#endif
+
+/**
+ * Whether async should be used by log messages, excluding error messages that are always sent sync.
+ **/
+#ifndef LOG_ASYNC_ENABLED
+    #define LOG_ASYNC_ENABLED YES
+#endif
+
+/**
+ * This is the single macro that all other macros below compile into.
+ * This big multiline macro makes all the other macros easier to read.
+ **/
+#define LOGV_MACRO(isAsynchronous, lvl, flg, ctx, atag, fnct, frmt, avalist) \
+        [DDLog log : isAsynchronous                                     \
+             level : lvl                                                \
+              flag : flg                                                \
+           context : ctx                                                \
+              file : __FILE__                                           \
+          function : fnct                                               \
+              line : __LINE__                                           \
+               tag : atag                                               \
+            format : frmt                                               \
+              args : avalist]
+
+/**
+ * Define version of the macro that only execute if the log level is above the threshold.
+ * The compiled versions essentially look like this:
+ *
+ * if (logFlagForThisLogMsg & ddLogLevel) { execute log message }
+ *
+ * When LOG_LEVEL_DEF is defined as ddLogLevel.
+ *
+ * As shown further below, Lumberjack actually uses a bitmask as opposed to primitive log levels.
+ * This allows for a great amount of flexibility and some pretty advanced fine grained logging techniques.
+ *
+ * Note that when compiler optimizations are enabled (as they are for your release builds),
+ * the log messages above your logging threshold will automatically be compiled out.
+ *
+ * (If the compiler sees LOG_LEVEL_DEF/ddLogLevel declared as a constant, the compiler simply checks to see
+ *  if the 'if' statement would execute, and if not it strips it from the binary.)
+ *
+ * We also define shorthand versions for asynchronous and synchronous logging.
+ **/
+#define LOGV_MAYBE(async, lvl, flg, ctx, tag, fnct, frmt, avalist) \
+        do { if(lvl & flg) LOGV_MACRO(async, lvl, flg, ctx, tag, fnct, frmt, avalist); } while(0)
+
+/**
+ * Ready to use log macros with no context or tag.
+ **/
+#define DDLogVError(frmt, avalist)   LOGV_MAYBE(NO,                LOG_LEVEL_DEF, DDLogFlagError,   0, nil, __PRETTY_FUNCTION__, frmt, avalist)
+#define DDLogVWarn(frmt, avalist)    LOGV_MAYBE(LOG_ASYNC_ENABLED, LOG_LEVEL_DEF, DDLogFlagWarning, 0, nil, __PRETTY_FUNCTION__, frmt, avalist)
+#define DDLogVInfo(frmt, avalist)    LOGV_MAYBE(LOG_ASYNC_ENABLED, LOG_LEVEL_DEF, DDLogFlagInfo,    0, nil, __PRETTY_FUNCTION__, frmt, avalist)
+#define DDLogVDebug(frmt, avalist)   LOGV_MAYBE(LOG_ASYNC_ENABLED, LOG_LEVEL_DEF, DDLogFlagDebug,   0, nil, __PRETTY_FUNCTION__, frmt, avalist)
+#define DDLogVVerbose(frmt, avalist) LOGV_MAYBE(LOG_ASYNC_ENABLED, LOG_LEVEL_DEF, DDLogFlagVerbose, 0, nil, __PRETTY_FUNCTION__, frmt, avalist)
+

CocoaLumberjack/DDLog.h

@@ -0,0 +1,545 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+#import <Foundation/Foundation.h>
+
+// Enable 1.9.x legacy macros if imported directly
+#ifndef DD_LEGACY_MACROS
+    #define DD_LEGACY_MACROS 1
+#endif
+// DD_LEGACY_MACROS is checked in the file itself
+#import "DDLegacyMacros.h"
+
+#if OS_OBJECT_USE_OBJC
+    #define DISPATCH_QUEUE_REFERENCE_TYPE strong
+#else
+    #define DISPATCH_QUEUE_REFERENCE_TYPE assign
+#endif
+
+@class DDLogMessage;
+@protocol DDLogger;
+@protocol DDLogFormatter;
+
+/**
+ * Define the standard options.
+ *
+ * We default to only 4 levels because it makes it easier for beginners
+ * to make the transition to a logging framework.
+ *
+ * More advanced users may choose to completely customize the levels (and level names) to suite their needs.
+ * For more information on this see the "Custom Log Levels" page:
+ * Documentation/CustomLogLevels.md
+ *
+ * Advanced users may also notice that we're using a bitmask.
+ * This is to allow for custom fine grained logging:
+ * Documentation/FineGrainedLogging.md
+ *
+ * -- Flags --
+ *
+ * Typically you will use the LOG_LEVELS (see below), but the flags may be used directly in certain situations.
+ * For example, say you have a lot of warning log messages, and you wanted to disable them.
+ * However, you still needed to see your error and info log messages.
+ * You could accomplish that with the following:
+ *
+ * static const DDLogLevel ddLogLevel = DDLogFlagError | DDLogFlagInfo;
+ *
+ * When LOG_LEVEL_DEF is defined as ddLogLevel.
+ *
+ * Flags may also be consulted when writing custom log formatters,
+ * as the DDLogMessage class captures the individual flag that caused the log message to fire.
+ *
+ * -- Levels --
+ *
+ * Log levels are simply the proper bitmask of the flags.
+ *
+ * -- Booleans --
+ *
+ * The booleans may be used when your logging code involves more than one line.
+ * For example:
+ *
+ * if (LOG_VERBOSE) {
+ *     for (id sprocket in sprockets)
+ *         DDLogVerbose(@"sprocket: %@", [sprocket description])
+ * }
+ *
+ * -- Async --
+ *
+ * Defines the default asynchronous options.
+ * The default philosophy for asynchronous logging is very simple:
+ *
+ * Log messages with errors should be executed synchronously.
+ *     After all, an error just occurred. The application could be unstable.
+ *
+ * All other log messages, such as debug output, are executed asynchronously.
+ *     After all, if it wasn't an error, then it was just informational output,
+ *     or something the application was easily able to recover from.
+ *
+ * -- Changes --
+ *
+ * You are strongly discouraged from modifying this file.
+ * If you do, you make it more difficult on yourself to merge future bug fixes and improvements from the project.
+ * Instead, create your own MyLogging.h or ApplicationNameLogging.h or CompanyLogging.h
+ *
+ * For an example of customizing your logging experience, see the "Custom Log Levels" page:
+ * Documentation/CustomLogLevels.md
+ **/
+
+typedef NS_OPTIONS(NSUInteger, DDLogFlag) {
+    DDLogFlagError      = (1 << 0), // 0...00001
+    DDLogFlagWarning    = (1 << 1), // 0...00010
+    DDLogFlagInfo       = (1 << 2), // 0...00100
+    DDLogFlagDebug      = (1 << 3), // 0...01000
+    DDLogFlagVerbose    = (1 << 4)  // 0...10000
+};
+
+typedef NS_ENUM(NSUInteger, DDLogLevel) {
+    DDLogLevelOff       = 0,
+    DDLogLevelError     = (DDLogFlagError),                       // 0...00001
+    DDLogLevelWarning   = (DDLogLevelError   | DDLogFlagWarning), // 0...00011
+    DDLogLevelInfo      = (DDLogLevelWarning | DDLogFlagInfo),    // 0...00111
+    DDLogLevelDebug     = (DDLogLevelInfo    | DDLogFlagDebug),   // 0...01111
+    DDLogLevelVerbose   = (DDLogLevelDebug   | DDLogFlagVerbose), // 0...11111
+    DDLogLevelAll       = NSUIntegerMax                           // 1111....11111 (DDLogLevelVerbose plus any other flags)
+};
+
+/**
+ * The THIS_FILE macro gives you an NSString of the file name.
+ * For simplicity and clarity, the file name does not include the full path or file extension.
+ *
+ * For example: DDLogWarn(@"%@: Unable to find thingy", THIS_FILE) -> @"MyViewController: Unable to find thingy"
+ **/
+
+NSString * DDExtractFileNameWithoutExtension(const char *filePath, BOOL copy);
+
+#define THIS_FILE         (DDExtractFileNameWithoutExtension(__FILE__, NO))
+
+/**
+ * The THIS_METHOD macro gives you the name of the current objective-c method.
+ *
+ * For example: DDLogWarn(@"%@ - Requires non-nil strings", THIS_METHOD) -> @"setMake:model: requires non-nil strings"
+ *
+ * Note: This does NOT work in straight C functions (non objective-c).
+ * Instead you should use the predefined __FUNCTION__ macro.
+ **/
+
+#define THIS_METHOD       NSStringFromSelector(_cmd)
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark -
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+@interface DDLog : NSObject
+
+/**
+ * Provides access to the underlying logging queue.
+ * This may be helpful to Logger classes for things like thread synchronization.
+ **/
+
++ (dispatch_queue_t)loggingQueue;
+
+/**
+ * Logging Primitive.
+ *
+ * This method is used by the macros above.
+ * It is suggested you stick with the macros as they're easier to use.
+ **/
+
++ (void)log:(BOOL)synchronous
+      level:(DDLogLevel)level
+       flag:(DDLogFlag)flag
+    context:(NSInteger)context
+       file:(const char *)file
+   function:(const char *)function
+       line:(NSUInteger)line
+        tag:(id)tag
+     format:(NSString *)format, ... NS_FORMAT_FUNCTION(9,10);
+
+/**
+ * Logging Primitive.
+ *
+ * This method can be used if you have a prepared va_list.
+ **/
+
++ (void)log:(BOOL)asynchronous
+      level:(DDLogLevel)level
+       flag:(DDLogFlag)flag
+    context:(NSInteger)context
+       file:(const char *)file
+   function:(const char *)function
+       line:(NSUInteger)line
+        tag:(id)tag
+     format:(NSString *)format
+       args:(va_list)argList;
+
+/**
+ * Logging Primitive.
+ **/
++ (void)log:(BOOL)asynchronous
+    message:(NSString *)message
+      level:(DDLogLevel)level
+       flag:(DDLogFlag)flag
+    context:(NSInteger)context
+       file:(const char *)file
+   function:(const char *)function
+       line:(NSUInteger)line
+        tag:(id)tag;
+
+/**
+ * Logging Primitive.
+ *
+ * This method can be used if you manualy prepared DDLogMessage.
+ **/
+
++ (void)log:(BOOL)asynchronous
+    message:(DDLogMessage *)logMessage;
+
+/**
+ * Since logging can be asynchronous, there may be times when you want to flush the logs.
+ * The framework invokes this automatically when the application quits.
+ **/
+
++ (void)flushLog;
+
+/**
+ * Loggers
+ *
+ * In order for your log statements to go somewhere, you should create and add a logger.
+ *
+ * You can add multiple loggers in order to direct your log statements to multiple places.
+ * And each logger can be configured separately.
+ * So you could have, for example, verbose logging to the console, but a concise log file with only warnings & errors.
+ **/
+
+/**
+ * Adds the logger to the system.
+ *
+ * This is equivalent to invoking [DDLog addLogger:logger withLogLevel:DDLogLevelAll].
+ **/
++ (void)addLogger:(id <DDLogger>)logger;
+
+/**
+ * Adds the logger to the system.
+ *
+ * The level that you provide here is a preemptive filter (for performance).
+ * That is, the level specified here will be used to filter out logMessages so that
+ * the logger is never even invoked for the messages.
+ *
+ * More information:
+ * When you issue a log statement, the logging framework iterates over each logger,
+ * and checks to see if it should forward the logMessage to the logger.
+ * This check is done using the level parameter passed to this method.
+ *
+ * For example:
+ *
+ * [DDLog addLogger:consoleLogger withLogLevel:DDLogLevelVerbose];
+ * [DDLog addLogger:fileLogger    withLogLevel:DDLogLevelWarning];
+ *
+ * DDLogError(@"oh no"); => gets forwarded to consoleLogger & fileLogger
+ * DDLogInfo(@"hi");     => gets forwarded to consoleLogger only
+ *
+ * It is important to remember that Lumberjack uses a BITMASK.
+ * Many developers & third party frameworks may define extra log levels & flags.
+ * For example:
+ *
+ * #define SOME_FRAMEWORK_LOG_FLAG_TRACE (1 << 6) // 0...1000000
+ *
+ * So if you specify DDLogLevelVerbose to this method, you won't see the framework's trace messages.
+ *
+ * (SOME_FRAMEWORK_LOG_FLAG_TRACE & DDLogLevelVerbose) => (01000000 & 00011111) => NO
+ *
+ * Consider passing DDLogLevelAll to this method, which has all bits set.
+ * You can also use the exclusive-or bitwise operator to get a bitmask that has all flags set,
+ * except the ones you explicitly don't want. For example, if you wanted everything except verbose & debug:
+ *
+ * ((DDLogLevelAll ^ DDLogLevelVerbose) | DDLogLevelInfo)
+ **/
++ (void)addLogger:(id <DDLogger>)logger withLevel:(DDLogLevel)level;
+
++ (void)removeLogger:(id <DDLogger>)logger;
++ (void)removeAllLoggers;
+
++ (NSArray *)allLoggers;
+
+/**
+ * Registered Dynamic Logging
+ *
+ * These methods allow you to obtain a list of classes that are using registered dynamic logging,
+ * and also provides methods to get and set their log level during run time.
+ **/
+
++ (NSArray *)registeredClasses;
++ (NSArray *)registeredClassNames;
+
++ (DDLogLevel)levelForClass:(Class)aClass;
++ (DDLogLevel)levelForClassWithName:(NSString *)aClassName;
+
++ (void)setLevel:(DDLogLevel)level forClass:(Class)aClass;
++ (void)setLevel:(DDLogLevel)level forClassWithName:(NSString *)aClassName;
+
+@end
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark -
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+@protocol DDLogger <NSObject>
+
+- (void)logMessage:(DDLogMessage *)logMessage;
+
+/**
+ * Formatters may optionally be added to any logger.
+ *
+ * If no formatter is set, the logger simply logs the message as it is given in logMessage,
+ * or it may use its own built in formatting style.
+ **/
+@property (nonatomic, strong) id <DDLogFormatter> logFormatter;
+
+@optional
+
+/**
+ * Since logging is asynchronous, adding and removing loggers is also asynchronous.
+ * In other words, the loggers are added and removed at appropriate times with regards to log messages.
+ *
+ * - Loggers will not receive log messages that were executed prior to when they were added.
+ * - Loggers will not receive log messages that were executed after they were removed.
+ *
+ * These methods are executed in the logging thread/queue.
+ * This is the same thread/queue that will execute every logMessage: invocation.
+ * Loggers may use these methods for thread synchronization or other setup/teardown tasks.
+ **/
+- (void)didAddLogger;
+- (void)willRemoveLogger;
+
+/**
+ * Some loggers may buffer IO for optimization purposes.
+ * For example, a database logger may only save occasionaly as the disk IO is slow.
+ * In such loggers, this method should be implemented to flush any pending IO.
+ *
+ * This allows invocations of DDLog's flushLog method to be propogated to loggers that need it.
+ *
+ * Note that DDLog's flushLog method is invoked automatically when the application quits,
+ * and it may be also invoked manually by the developer prior to application crashes, or other such reasons.
+ **/
+- (void)flush;
+
+/**
+ * Each logger is executed concurrently with respect to the other loggers.
+ * Thus, a dedicated dispatch queue is used for each logger.
+ * Logger implementations may optionally choose to provide their own dispatch queue.
+ **/
+@property (nonatomic, DISPATCH_QUEUE_REFERENCE_TYPE, readonly) dispatch_queue_t loggerQueue;
+
+/**
+ * If the logger implementation does not choose to provide its own queue,
+ * one will automatically be created for it.
+ * The created queue will receive its name from this method.
+ * This may be helpful for debugging or profiling reasons.
+ **/
+@property (nonatomic, readonly) NSString *loggerName;
+
+@end
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark -
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+@protocol DDLogFormatter <NSObject>
+@required
+
+/**
+ * Formatters may optionally be added to any logger.
+ * This allows for increased flexibility in the logging environment.
+ * For example, log messages for log files may be formatted differently than log messages for the console.
+ *
+ * For more information about formatters, see the "Custom Formatters" page:
+ * Documentation/CustomFormatters.md
+ *
+ * The formatter may also optionally filter the log message by returning nil,
+ * in which case the logger will not log the message.
+ **/
+- (NSString *)formatLogMessage:(DDLogMessage *)logMessage;
+
+@optional
+
+/**
+ * A single formatter instance can be added to multiple loggers.
+ * These methods provides hooks to notify the formatter of when it's added/removed.
+ *
+ * This is primarily for thread-safety.
+ * If a formatter is explicitly not thread-safe, it may wish to throw an exception if added to multiple loggers.
+ * Or if a formatter has potentially thread-unsafe code (e.g. NSDateFormatter),
+ * it could possibly use these hooks to switch to thread-safe versions of the code.
+ **/
+- (void)didAddToLogger:(id <DDLogger>)logger;
+- (void)willRemoveFromLogger:(id <DDLogger>)logger;
+
+@end
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark -
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+@protocol DDRegisteredDynamicLogging
+
+/**
+ * Implement these methods to allow a file's log level to be managed from a central location.
+ *
+ * This is useful if you'd like to be able to change log levels for various parts
+ * of your code from within the running application.
+ *
+ * Imagine pulling up the settings for your application,
+ * and being able to configure the logging level on a per file basis.
+ *
+ * The implementation can be very straight-forward:
+ *
+ * + (int)ddLogLevel
+ * {
+ *     return ddLogLevel;
+ * }
+ *
+ * + (void)ddSetLogLevel:(DDLogLevel)level
+ * {
+ *     ddLogLevel = level;
+ * }
+ **/
+
++ (DDLogLevel)ddLogLevel;
++ (void)ddSetLogLevel:(DDLogLevel)level;
+
+@end
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark -
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#ifndef NS_DESIGNATED_INITIALIZER
+    #define NS_DESIGNATED_INITIALIZER
+#endif
+
+typedef NS_OPTIONS(NSInteger, DDLogMessageOptions) {
+    DDLogMessageCopyFile     = 1 << 0,
+    DDLogMessageCopyFunction = 1 << 1
+};
+
+/**
+ * The DDLogMessage class encapsulates information about the log message.
+ * If you write custom loggers or formatters, you will be dealing with objects of this class.
+ **/
+
+@interface DDLogMessage : NSObject <NSCopying>
+{
+    // Direct accessors to be used only for performance
+    @public
+    NSString *_message;
+    DDLogLevel _level;
+    DDLogFlag _flag;
+    NSInteger _context;
+    NSString *_file;
+    NSString *_fileName;
+    NSString *_function;
+    NSUInteger _line;
+    id _tag;
+    DDLogMessageOptions _options;
+    NSDate *_timestamp;
+    NSString *_threadID;
+    NSString *_threadName;
+    NSString *_queueLabel;
+}
+
+/**
+ * Standard init method for a log message object.
+ * Used by the logging primitives. (And the macros use the logging primitives.)
+ *
+ * If you find need to manually create logMessage objects, there is one thing you should be aware of:
+ *
+ * If no flags are passed, the method expects the file and function parameters to be string literals.
+ * That is, it expects the given strings to exist for the duration of the object's lifetime,
+ * and it expects the given strings to be immutable.
+ * In other words, it does not copy these strings, it simply points to them.
+ * This is due to the fact that __FILE__ and __FUNCTION__ are usually used to specify these parameters,
+ * so it makes sense to optimize and skip the unnecessary allocations.
+ * However, if you need them to be copied you may use the options parameter to specify this.
+ * Options is a bitmask which supports DDLogMessageCopyFile and DDLogMessageCopyFunction.
+ **/
+- (instancetype)initWithMessage:(NSString *)message
+                          level:(DDLogLevel)level
+                           flag:(DDLogFlag)flag
+                        context:(NSInteger)context
+                           file:(NSString *)file
+                       function:(NSString *)function
+                           line:(NSUInteger)line
+                            tag:(id)tag
+                        options:(DDLogMessageOptions)options
+                      timestamp:(NSDate *)timestamp NS_DESIGNATED_INITIALIZER;
+
+/**
+ * Read-only properties
+ **/
+@property (readonly, nonatomic) NSString *message;
+@property (readonly, nonatomic) DDLogLevel level;
+@property (readonly, nonatomic) DDLogFlag flag;
+@property (readonly, nonatomic) NSInteger context;
+@property (readonly, nonatomic) NSString *file;
+@property (readonly, nonatomic) NSString *fileName;
+@property (readonly, nonatomic) NSString *function;
+@property (readonly, nonatomic) NSUInteger line;
+@property (readonly, nonatomic) id tag;
+@property (readonly, nonatomic) DDLogMessageOptions options;
+@property (readonly, nonatomic) NSDate *timestamp;
+@property (readonly, nonatomic) NSString *threadID; // ID as it appears in NSLog calculated from the machThreadID
+@property (readonly, nonatomic) NSString *threadName;
+@property (readonly, nonatomic) NSString *queueLabel;
+
+@end
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+#pragma mark -
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * The DDLogger protocol specifies that an optional formatter can be added to a logger.
+ * Most (but not all) loggers will want to support formatters.
+ *
+ * However, writting getters and setters in a thread safe manner,
+ * while still maintaining maximum speed for the logging process, is a difficult task.
+ *
+ * To do it right, the implementation of the getter/setter has strict requiremenets:
+ * - Must NOT require the logMessage method to acquire a lock.
+ * - Must NOT require the logMessage method to access an atomic property (also a lock of sorts).
+ *
+ * To simplify things, an abstract logger is provided that implements the getter and setter.
+ *
+ * Logger implementations may simply extend this class,
+ * and they can ACCESS THE FORMATTER VARIABLE DIRECTLY from within their logMessage method!
+ **/
+
+@interface DDAbstractLogger : NSObject <DDLogger>
+{
+    // Direct accessors to be used only for performance
+    @public
+    id <DDLogFormatter> _logFormatter;
+    dispatch_queue_t _loggerQueue;
+}
+
+@property (nonatomic, strong) id <DDLogFormatter> logFormatter;
+@property (nonatomic, DISPATCH_QUEUE_REFERENCE_TYPE) dispatch_queue_t loggerQueue;
+
+// For thread-safety assertions
+@property (nonatomic, readonly, getter=isOnGlobalLoggingQueue)  BOOL onGlobalLoggingQueue;
+@property (nonatomic, readonly, getter=isOnInternalLoggerQueue) BOOL onInternalLoggerQueue;
+
+@end
+

CocoaLumberjack/DDLogMacros.h

@@ -0,0 +1,82 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+// Disable legacy macros
+#ifndef DD_LEGACY_MACROS
+    #define DD_LEGACY_MACROS 0
+#endif
+
+#import "DDLog.h"
+
+/**
+ * The constant/variable/method responsible for controlling the current log level.
+ **/
+#ifndef LOG_LEVEL_DEF
+    #define LOG_LEVEL_DEF ddLogLevel
+#endif
+
+/**
+ * Whether async should be used by log messages, excluding error messages that are always sent sync.
+ **/
+#ifndef LOG_ASYNC_ENABLED
+    #define LOG_ASYNC_ENABLED YES
+#endif
+
+/**
+ * This is the single macro that all other macros below compile into.
+ * This big multiline macro makes all the other macros easier to read.
+ **/
+#define LOG_MACRO(isAsynchronous, lvl, flg, ctx, atag, fnct, frmt, ...) \
+        [DDLog log : isAsynchronous                                     \
+             level : lvl                                                \
+              flag : flg                                                \
+           context : ctx                                                \
+              file : __FILE__                                           \
+          function : fnct                                               \
+              line : __LINE__                                           \
+               tag : atag                                               \
+            format : (frmt), ## __VA_ARGS__]
+
+/**
+ * Define version of the macro that only execute if the log level is above the threshold.
+ * The compiled versions essentially look like this:
+ *
+ * if (logFlagForThisLogMsg & ddLogLevel) { execute log message }
+ *
+ * When LOG_LEVEL_DEF is defined as ddLogLevel.
+ *
+ * As shown further below, Lumberjack actually uses a bitmask as opposed to primitive log levels.
+ * This allows for a great amount of flexibility and some pretty advanced fine grained logging techniques.
+ *
+ * Note that when compiler optimizations are enabled (as they are for your release builds),
+ * the log messages above your logging threshold will automatically be compiled out.
+ *
+ * (If the compiler sees LOG_LEVEL_DEF/ddLogLevel declared as a constant, the compiler simply checks to see
+ *  if the 'if' statement would execute, and if not it strips it from the binary.)
+ *
+ * We also define shorthand versions for asynchronous and synchronous logging.
+ **/
+#define LOG_MAYBE(async, lvl, flg, ctx, tag, fnct, frmt, ...) \
+        do { if(lvl & flg) LOG_MACRO(async, lvl, flg, ctx, tag, fnct, frmt, ##__VA_ARGS__); } while(0)
+
+/**
+ * Ready to use log macros with no context or tag.
+ **/
+#define DDLogError(frmt, ...)   LOG_MAYBE(NO,                LOG_LEVEL_DEF, DDLogFlagError,   0, nil, __PRETTY_FUNCTION__, frmt, ##__VA_ARGS__)
+#define DDLogWarn(frmt, ...)    LOG_MAYBE(LOG_ASYNC_ENABLED, LOG_LEVEL_DEF, DDLogFlagWarning, 0, nil, __PRETTY_FUNCTION__, frmt, ##__VA_ARGS__)
+#define DDLogInfo(frmt, ...)    LOG_MAYBE(LOG_ASYNC_ENABLED, LOG_LEVEL_DEF, DDLogFlagInfo,    0, nil, __PRETTY_FUNCTION__, frmt, ##__VA_ARGS__)
+#define DDLogDebug(frmt, ...)   LOG_MAYBE(LOG_ASYNC_ENABLED, LOG_LEVEL_DEF, DDLogFlagDebug,   0, nil, __PRETTY_FUNCTION__, frmt, ##__VA_ARGS__)
+#define DDLogVerbose(frmt, ...) LOG_MAYBE(LOG_ASYNC_ENABLED, LOG_LEVEL_DEF, DDLogFlagVerbose, 0, nil, __PRETTY_FUNCTION__, frmt, ##__VA_ARGS__)
+

CocoaLumberjack/DDMultiFormatter.h

@@ -0,0 +1,42 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+#import <Foundation/Foundation.h>
+
+// Disable legacy macros
+#ifndef DD_LEGACY_MACROS
+    #define DD_LEGACY_MACROS 0
+#endif
+
+#import "DDLog.h"
+
+/**
+ * This formatter can be used to chain different formatters together.
+ * The log message will processed in the order of the formatters added.
+ **/
+
+@interface DDMultiFormatter : NSObject <DDLogFormatter>
+
+/**
+ *  Array of chained formatters
+ */
+@property (readonly) NSArray *formatters;
+
+- (void)addFormatter:(id<DDLogFormatter>)formatter;
+- (void)removeFormatter:(id<DDLogFormatter>)formatter;
+- (void)removeAllFormatters;
+- (BOOL)isFormattingWithFormatter:(id<DDLogFormatter>)formatter;
+
+@end

CocoaLumberjack/DDMultiFormatter.m

@@ -0,0 +1,141 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+#import "DDMultiFormatter.h"
+
+
+#if TARGET_OS_IPHONE
+// Compiling for iOS
+#if __IPHONE_OS_VERSION_MIN_REQUIRED >= 60000 // iOS 6.0 or later
+#define NEEDS_DISPATCH_RETAIN_RELEASE 0
+#else                                         // iOS 5.X or earlier
+#define NEEDS_DISPATCH_RETAIN_RELEASE 1
+#endif
+#else
+// Compiling for Mac OS X
+#if MAC_OS_X_VERSION_MIN_REQUIRED >= 1080     // Mac OS X 10.8 or later
+#define NEEDS_DISPATCH_RETAIN_RELEASE 0
+#else                                         // Mac OS X 10.7 or earlier
+#define NEEDS_DISPATCH_RETAIN_RELEASE 1
+#endif
+#endif
+
+
+#if !__has_feature(objc_arc)
+#error This file must be compiled with ARC. Use -fobjc-arc flag (or convert project to ARC).
+#endif
+
+
+@interface DDMultiFormatter () {
+    dispatch_queue_t _queue;
+    NSMutableArray *_formatters;
+}
+
+- (DDLogMessage *)logMessageForLine:(NSString *)line originalMessage:(DDLogMessage *)message;
+
+@end
+
+
+@implementation DDMultiFormatter
+
+- (instancetype)init {
+    self = [super init];
+
+    if (self) {
+#if MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
+        _queue = dispatch_queue_create("cocoa.lumberjack.multiformatter", DISPATCH_QUEUE_CONCURRENT);
+#else
+        _queue = dispatch_queue_create("cocoa.lumberjack.multiformatter", NULL);
+#endif
+        _formatters = [NSMutableArray new];
+    }
+
+    return self;
+}
+
+#if NEEDS_DISPATCH_RETAIN_RELEASE
+- (void)dealloc {
+    dispatch_release(_queue);
+}
+
+#endif
+
+#pragma mark Processing
+
+- (NSString *)formatLogMessage:(DDLogMessage *)logMessage {
+    __block NSString *line = logMessage->_message;
+
+    dispatch_sync(_queue, ^{
+        for (id<DDLogFormatter> formatter in _formatters) {
+            DDLogMessage *message = [self logMessageForLine:line originalMessage:logMessage];
+            line = [formatter formatLogMessage:message];
+
+            if (!line) {
+                break;
+            }
+        }
+    });
+
+    return line;
+}
+
+- (DDLogMessage *)logMessageForLine:(NSString *)line originalMessage:(DDLogMessage *)message {
+    DDLogMessage *newMessage = [message copy];
+
+    newMessage->_message = line;
+    return newMessage;
+}
+
+#pragma mark Formatters
+
+- (NSArray *)formatters {
+    __block NSArray *formatters;
+
+    dispatch_sync(_queue, ^{
+        formatters = [_formatters copy];
+    });
+
+    return formatters;
+}
+
+- (void)addFormatter:(id<DDLogFormatter>)formatter {
+    dispatch_barrier_async(_queue, ^{
+        [_formatters addObject:formatter];
+    });
+}
+
+- (void)removeFormatter:(id<DDLogFormatter>)formatter {
+    dispatch_barrier_async(_queue, ^{
+        [_formatters removeObject:formatter];
+    });
+}
+
+- (void)removeAllFormatters {
+    dispatch_barrier_async(_queue, ^{
+        [_formatters removeAllObjects];
+    });
+}
+
+- (BOOL)isFormattingWithFormatter:(id<DDLogFormatter>)formatter {
+    __block BOOL hasFormatter;
+
+    dispatch_sync(_queue, ^{
+        hasFormatter = [_formatters containsObject:formatter];
+    });
+
+    return hasFormatter;
+}
+
+@end

CocoaLumberjack/DDTTYLogger.h

@@ -0,0 +1,176 @@
+// Software License Agreement (BSD License)
+//
+// Copyright (c) 2010-2015, Deusty, LLC
+// All rights reserved.
+//
+// Redistribution and use of this software in source and binary forms,
+// with or without modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Neither the name of Deusty nor the names of its contributors may be used
+//   to endorse or promote products derived from this software without specific
+//   prior written permission of Deusty, LLC.
+
+/**
+ * This class provides a logger for Terminal output or Xcode console output,
+ * depending on where you are running your code.
+ *
+ * As described in the "Getting Started" page,
+ * the traditional NSLog() function directs it's output to two places:
+ *
+ * - Apple System Log (so it shows up in Console.app)
+ * - StdErr (if stderr is a TTY, so log statements show up in Xcode console)
+ *
+ * To duplicate NSLog() functionality you can simply add this logger and an asl logger.
+ * However, if you instead choose to use file logging (for faster performance),
+ * you may choose to use only a file logger and a tty logger.
+ **/
+
+// Disable legacy macros
+#ifndef DD_LEGACY_MACROS
+    #define DD_LEGACY_MACROS 0
+#endif
+
+#import "DDLog.h"
+
+#define LOG_CONTEXT_ALL INT_MAX
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunused-function"
+#if TARGET_OS_IPHONE
+    // iOS
+    #import <UIKit/UIColor.h>
+    typedef UIColor DDColor;
+    static  DDColor* DDMakeColor(CGFloat r, CGFloat g, CGFloat b) {return [DDColor colorWithRed:(r/255.0f) green:(g/255.0f) blue:(b/255.0f) alpha:1.0f];}
+#elif __has_include(<AppKit/NSColor.h>)
+    // OS X with AppKit
+    #import <AppKit/NSColor.h>
+    typedef NSColor DDColor;
+    static DDColor* DDMakeColor(CGFloat r, CGFloat g, CGFloat b) {return [DDColor colorWithCalibratedRed:(r/255.0f) green:(g/255.0f) blue:(b/255.0f) alpha:1.0f];}
+#else
+    // OS X CLI
+    #import "CLIColor.h"
+    typedef CLIColor DDColor;
+    static  DDColor* DDMakeColor(CGFloat r, CGFloat g, CGFloat b) {return [DDColor colorWithCalibratedRed:(r/255.0f) green:(g/255.0f) blue:(b/255.0f) alpha:1.0f];}
+#endif
+#pragma clang diagnostic pop
+
+@interface DDTTYLogger : DDAbstractLogger <DDLogger>
+
++ (instancetype)sharedInstance;
+
+/* Inherited from the DDLogger protocol:
+ *
+ * Formatters may optionally be added to any logger.
+ *
+ * If no formatter is set, the logger simply logs the message as it is given in logMessage,
+ * or it may use its own built in formatting style.
+ *
+ * More information about formatters can be found here:
+ * Documentation/CustomFormatters.md
+ *
+ * The actual implementation of these methods is inherited from DDAbstractLogger.
+
+   - (id <DDLogFormatter>)logFormatter;
+   - (void)setLogFormatter:(id <DDLogFormatter>)formatter;
+
+ */
+
+/**
+ * Want to use different colors for different log levels?
+ * Enable this property.
+ *
+ * If you run the application via the Terminal (not Xcode),
+ * the logger will map colors to xterm-256color or xterm-color (if available).
+ *
+ * Xcode does NOT natively support colors in the Xcode debugging console.
+ * You'll need to install the XcodeColors plugin to see colors in the Xcode console.
+ * https://github.com/robbiehanson/XcodeColors
+ *
+ * The default value is NO.
+ **/
+@property (readwrite, assign) BOOL colorsEnabled;
+
+/**
+ * When using a custom formatter you can set the logMessage method not to append
+ * '\n' character after each output. This allows for some greater flexibility with
+ * custom formatters. Default value is YES.
+ **/
+
+@property (nonatomic, readwrite, assign) BOOL automaticallyAppendNewlineForCustomFormatters;
+
+/**
+ * The default color set (foregroundColor, backgroundColor) is:
+ *
+ * - DDLogFlagError   = (red, nil)
+ * - DDLogFlagWarning = (orange, nil)
+ *
+ * You can customize the colors however you see fit.
+ * Please note that you are passing a flag, NOT a level.
+ *
+ * GOOD : [ttyLogger setForegroundColor:pink backgroundColor:nil forFlag:DDLogFlagInfo];  // <- Good :)
+ *  BAD : [ttyLogger setForegroundColor:pink backgroundColor:nil forFlag:DDLogLevelInfo]; // <- BAD! :(
+ *
+ * DDLogFlagInfo  = 0...00100
+ * DDLogLevelInfo = 0...00111 <- Would match DDLogFlagInfo and DDLogFlagWarning and DDLogFlagError
+ *
+ * If you run the application within Xcode, then the XcodeColors plugin is required.
+ *
+ * If you run the application from a shell, then DDTTYLogger will automatically map the given color to
+ * the closest available color. (xterm-256color or xterm-color which have 256 and 16 supported colors respectively.)
+ *
+ * This method invokes setForegroundColor:backgroundColor:forFlag:context: and applies it to `LOG_CONTEXT_ALL`.
+ **/
+- (void)setForegroundColor:(DDColor *)txtColor backgroundColor:(DDColor *)bgColor forFlag:(DDLogFlag)mask;
+
+/**
+ * Just like setForegroundColor:backgroundColor:flag, but allows you to specify a particular logging context.
+ *
+ * A logging context is often used to identify log messages coming from a 3rd party framework,
+ * although logging context's can be used for many different functions.
+ *
+ * Use LOG_CONTEXT_ALL to set the deafult color for all contexts that have no specific color set defined.
+ *
+ * Logging context's are explained in further detail here:
+ * Documentation/CustomContext.md
+ **/
+- (void)setForegroundColor:(DDColor *)txtColor backgroundColor:(DDColor *)bgColor forFlag:(DDLogFlag)mask context:(NSInteger)ctxt;
+
+/**
+ * Similar to the methods above, but allows you to map DDLogMessage->tag to a particular color profile.
+ * For example, you could do something like this:
+ *
+ * static NSString *const PurpleTag = @"PurpleTag";
+ *
+ * #define DDLogPurple(frmt, ...) LOG_OBJC_TAG_MACRO(NO, 0, 0, 0, PurpleTag, frmt, ##__VA_ARGS__)
+ * 
+ * And then where you configure CocoaLumberjack:
+ *
+ * purple = DDMakeColor((64/255.0), (0/255.0), (128/255.0));
+ *
+ * or any UIColor/NSColor constructor.
+ *
+ * Note: For CLI OS X projects that don't link with AppKit use CLIColor objects instead
+ *
+ * [[DDTTYLogger sharedInstance] setForegroundColor:purple backgroundColor:nil forTag:PurpleTag];
+ * [DDLog addLogger:[DDTTYLogger sharedInstance]];
+ *
+ * This would essentially give you a straight NSLog replacement that prints in purple:
+ *
+ * DDLogPurple(@"I'm a purple log message!");
+ **/
+- (void)setForegroundColor:(DDColor *)txtColor backgroundColor:(DDColor *)bgColor forTag:(id <NSCopying>)tag;
+
+/**
+ * Clearing color profiles.
+ **/
+- (void)clearColorsForFlag:(DDLogFlag)mask;
+- (void)clearColorsForFlag:(DDLogFlag)mask context:(NSInteger)context;
+- (void)clearColorsForTag:(id <NSCopying>)tag;
+- (void)clearColorsForAllFlags;
+- (void)clearColorsForAllTags;
+- (void)clearAllColors;
+
+@end

DeleteDelegate.h

@@ -0,0 +1,39 @@
+/*
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
+ 
+ All rights reserved.
+ 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ 
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 
+ * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
+ their contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+
+@protocol DeleteDelegate <NSObject>
+- (BOOL)didDelete:(NSUInteger)count ofTotal:(NSUInteger)total error:(NSError **)error;
+- (void)didFinishDeleting:(NSUInteger)total;
+@end

GoogleDriveTargetConnection.m

@@ -1,127 +0,0 @@
-/*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#import "GoogleDriveTargetConnection.h"
-#import "GoogleDriveRemoteFS.h"
-#import "BaseTargetConnection.h"
-#import "S3ObjectMetadata.h"
-
-
-@implementation GoogleDriveTargetConnection
-- (id)initWithTarget:(Target *)theTarget {
-    if (self = [super init]) {
-        googleDriveRemoteFS = [[GoogleDriveRemoteFS alloc] initWithTarget:theTarget];
-        base = [[BaseTargetConnection alloc] initWithTarget:theTarget remoteFS:googleDriveRemoteFS];
-    }
-    return self;
-}
-- (void)dealloc {
-    [googleDriveRemoteFS release];
-    [base release];
-    [super dealloc];
-}
-
-
-#pragma mark TargetConnection
-- (NSArray *)computerUUIDsWithDelegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base computerUUIDsWithDelegate:theDelegate error:error];
-}
-- (NSArray *)bucketUUIDsForComputerUUID:(NSString *)theComputerUUID deleted:(BOOL)deleted delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base bucketUUIDsForComputerUUID:theComputerUUID deleted:deleted delegate:theDelegate error:error];
-}
-- (NSData *)bucketPlistDataForComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base bucketPlistDataForComputerUUID:theComputerUUID bucketUUID:theBucketUUID deleted:deleted delegate:theDelegate error:error];
-}
-- (BOOL)saveBucketPlistData:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base saveBucketPlistData:theData forComputerUUID:theComputerUUID bucketUUID:theBucketUUID deleted:deleted delegate:theDelegate error:error];
-}
-- (BOOL)deleteBucketPlistDataForComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base deleteBucketPlistDataForComputerUUID:theComputerUUID bucketUUID:theBucketUUID deleted:deleted delegate:theDelegate error:error];
-}
-- (NSData *)computerInfoForComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base computerInfoForComputerUUID:theComputerUUID delegate:theDelegate error:error];
-}
-- (BOOL)saveComputerInfo:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base saveComputerInfo:theData forComputerUUID:theComputerUUID delegate:theDelegate error:error];
-}
-- (NSDictionary *)objectsBySHA1ForTargetEndpoint:(NSURL *)theEndpoint isGlacier:(BOOL)theIsGlacier computerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    NSString *prefix = [NSString stringWithFormat:@"%@/%@/objects/", [theEndpoint path], theComputerUUID];
-    
-    NSArray *objects = [base objectsWithPrefix:prefix delegate:theDelegate error:error];
-    if (objects == nil) {
-        return nil;
-    }
-    NSMutableDictionary *ret = [NSMutableDictionary dictionary];
-    for (S3ObjectMetadata *md in objects) {
-        [ret setObject:md forKey:[[md path] lastPathComponent]];
-    }
-    return ret;
-}
-- (NSArray *)pathsWithPrefix:(NSString *)thePrefix delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base pathsWithPrefix:thePrefix delegate:theDelegate error:error];
-}
-- (BOOL)deleteObjectsForComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base deleteObjectsForComputerUUID:theComputerUUID delegate:theDelegate error:error];
-}
-- (BOOL)deletePaths:(NSArray *)thePaths delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base deletePaths:thePaths delegate:theDelegate error:error];
-}
-- (NSNumber *)fileExistsAtPath:(NSString *)thePath dataSize:(unsigned long long *)theDataSize delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base fileExistsAtPath:thePath dataSize:theDataSize delegate:theDelegate error:error];
-}
-- (NSData *)contentsOfFileAtPath:(NSString *)thePath delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base contentsOfFileAtPath:thePath delegate:theDelegate error:error];
-}
-- (BOOL)writeData:(NSData *)theData toFileAtPath:(NSString *)thePath dataTransferDelegate:(id <DataTransferDelegate>)theDataTransferDelegate targetConnectionDelegate:(id<TargetConnectionDelegate>)theTargetConnectionDelegate error:(NSError **)error {
-    return [base writeData:theData toFileAtPath:thePath dataTransferDelegate:theDataTransferDelegate targetConnectionDelegate:theTargetConnectionDelegate error:error];
-}
-- (BOOL)removeItemAtPath:(NSString *)thePath delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base removeItemAtPath:thePath delegate:theDelegate error:error];
-}
-- (NSNumber *)sizeOfItemAtPath:(NSString *)thePath delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base sizeOfItemAtPath:thePath delegate:theDelegate error:error];
-}
-- (NSNumber *)isObjectRestoredAtPath:(NSString *)thePath delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base isObjectRestoredAtPath:thePath delegate:theDelegate error:error];
-}
-- (BOOL)restoreObjectAtPath:(NSString *)thePath forDays:(NSUInteger)theDays alreadyRestoredOrRestoring:(BOOL *)alreadyRestoredOrRestoring delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base restoreObjectAtPath:thePath forDays:theDays alreadyRestoredOrRestoring:alreadyRestoredOrRestoring delegate:theDelegate error:error];
-}
-- (NSData *)saltDataForComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base saltDataForComputerUUID:theComputerUUID delegate:theDelegate error:error];
-}
-- (BOOL)setSaltData:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base setSaltData:theData forComputerUUID:theComputerUUID delegate:theDelegate error:error];
-}
-
-@end

NSString_slashed.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -32,6 +32,7 @@
 
 
 
+
 @interface NSString (slashed)
 - (NSString *)slashed;
 @end

NSString_slashed.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,6 +31,8 @@
  */
 
 
+
+
 #import "NSString_slashed.h"
 
 @implementation NSString (slashed)

ReflogEntry.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,18 +31,26 @@
  */
 
 
-#import "CWLSynthesizeSingleton.h"
+
-#import "GoogleDrive.h"
+@class DictNode;
+@class BlobKey;
 
 
-@interface GoogleDriveFactory : NSObject <GoogleDriveDelegate> {
+@interface ReflogEntry : NSObject {
-    NSMutableDictionary *accessTokensByRefreshToken;
+    NSDate *createdDate;
-    NSMutableDictionary *folderIdDictionariesByRefreshToken;
+    NSString *reflogId;
-    NSLock *lock;
+    BlobKey *oldHeadBlobKey;
+    BlobKey *newHeadBlobKey;
+    BOOL isRewrite;
+    NSString *packSHA1;
 }
 
-CWL_DECLARE_SINGLETON_FOR_CLASS(GoogleDriveFactory);
+- (id)initWithId:(NSString *)theId plist:(DictNode *)thePlist error:(NSError **)error;
 
-
+- (NSDate *)createdDate;
-- (GoogleDrive *)googleDriveWithEmailAddress:(NSString *)theEmailAddress refreshToken:(NSString *)theRefreshToken;
+- (NSString *)reflogId;
+- (BlobKey *)oldHeadBlobKey;
+- (BlobKey *)newHeadBlobKey;
+- (BOOL)isRewrite;
+- (NSString *)packSHA1;
 @end

ReflogEntry.m

@@ -0,0 +1,99 @@
+/*
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
+ 
+ All rights reserved.
+ 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ 
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 
+ * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
+ their contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+#import "ReflogEntry.h"
+#import "DictNode.h"
+#import "BlobKey.h"
+#import "BooleanNode.h"
+#import "StringNode.h"
+
+
+@implementation ReflogEntry
+- (id)initWithId:(NSString *)theReflogId plist:(DictNode *)thePlist error:(NSError **)error {
+    if (self = [super init]) {
+        reflogId = [theReflogId retain];
+        double timeInterval = [reflogId doubleValue];
+        createdDate = [[NSDate alloc] initWithTimeIntervalSinceReferenceDate:timeInterval];
+        
+        if ([thePlist containsKey:@"oldHeadSHA1"]) {
+            BOOL oldHeadStretchKey = [[thePlist booleanNodeForKey:@"oldHeadStretchKey"] booleanValue];
+            NSString *oldHeadSHA1 = [[thePlist stringNodeForKey:@"oldHeadSHA1"] stringValue];
+            oldHeadBlobKey = [[BlobKey alloc] initWithSHA1:oldHeadSHA1 storageType:StorageTypeS3 stretchEncryptionKey:oldHeadStretchKey compressionType:BlobKeyCompressionNone error:error];
+            if (oldHeadBlobKey == nil) {
+                [self release];
+                return nil;
+            }
+        }
+        BOOL newHeadStretchKey = [[thePlist booleanNodeForKey:@"newHeadStretchKey"] booleanValue];
+        NSString *newHeadSHA1 = [[thePlist stringNodeForKey:@"newHeadSHA1"] stringValue];
+        newHeadBlobKey = [[BlobKey alloc] initWithSHA1:newHeadSHA1 storageType:StorageTypeS3 stretchEncryptionKey:newHeadStretchKey compressionType:BlobKeyCompressionNone error:error];
+        if (newHeadBlobKey == nil) {
+            [self release];
+            return nil;
+        }
+        
+        isRewrite = [[thePlist booleanNodeForKey:@"isRewrite"] booleanValue];
+        
+        packSHA1 = [[[thePlist stringNodeForKey:@"packSHA1"] stringValue] retain];
+    }
+    return self;
+}
+- (void)dealloc {
+    [createdDate release];
+    [reflogId release];
+    [oldHeadBlobKey release];
+    [newHeadBlobKey release];
+    [packSHA1 release];
+    [super dealloc];
+}
+
+- (NSDate *)createdDate {
+    return createdDate;
+}
+- (NSString *)reflogId {
+    return reflogId;
+}
+- (BlobKey *)oldHeadBlobKey {
+    return oldHeadBlobKey;
+}
+- (BlobKey *)newHeadBlobKey {
+    return newHeadBlobKey;
+}
+- (BOOL)isRewrite {
+    return isRewrite;
+}
+- (NSString *)packSHA1 {
+    return packSHA1;
+}
+@end

S3TargetConnection.m

@@ -1,126 +0,0 @@
-/*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#import "S3TargetConnection.h"
-#import "BaseTargetConnection.h"
-#import "S3RemoteFS.h"
-#import "S3ObjectMetadata.h"
-
-
-@implementation S3TargetConnection
-- (id)initWithTarget:(Target *)theTarget {
-    if (self = [super init]) {
-        s3RemoteFS = [[S3RemoteFS alloc] initWithTarget:theTarget];
-        base = [[BaseTargetConnection alloc] initWithTarget:theTarget remoteFS:s3RemoteFS];
-    }
-    return self;
-}
-- (void)dealloc {
-    [s3RemoteFS release];
-    [base release];
-    [super dealloc];
-}
-
-
-#pragma mark TargetConnection
-- (NSArray *)computerUUIDsWithDelegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base computerUUIDsWithDelegate:theDelegate error:error];
-}
-- (NSArray *)bucketUUIDsForComputerUUID:(NSString *)theComputerUUID deleted:(BOOL)deleted delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base bucketUUIDsForComputerUUID:theComputerUUID deleted:deleted delegate:theDelegate error:error];
-}
-- (NSData *)bucketPlistDataForComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base bucketPlistDataForComputerUUID:theComputerUUID bucketUUID:theBucketUUID deleted:deleted delegate:theDelegate error:error];
-}
-- (BOOL)saveBucketPlistData:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base saveBucketPlistData:theData forComputerUUID:theComputerUUID bucketUUID:theBucketUUID deleted:deleted delegate:theDelegate error:error];
-}
-- (BOOL)deleteBucketPlistDataForComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base deleteBucketPlistDataForComputerUUID:theComputerUUID bucketUUID:theBucketUUID deleted:deleted delegate:theDelegate error:error];
-}
-- (NSData *)computerInfoForComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base computerInfoForComputerUUID:theComputerUUID delegate:theDelegate error:error];
-}
-- (BOOL)saveComputerInfo:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base saveComputerInfo:theData forComputerUUID:theComputerUUID delegate:theDelegate error:error];
-}
-- (NSDictionary *)objectsBySHA1ForTargetEndpoint:(NSURL *)theEndpoint isGlacier:(BOOL)theIsGlacier computerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    NSString *prefix = [NSString stringWithFormat:@"%@/%@%@/objects/", [theEndpoint path], (theIsGlacier ? @"glacier/" : @""), theComputerUUID];
-    
-    NSArray *objects = [base objectsWithPrefix:prefix delegate:theDelegate error:error];
-    if (objects == nil) {
-        return nil;
-    }
-    NSMutableDictionary *ret = [NSMutableDictionary dictionary];
-    for (S3ObjectMetadata *md in objects) {
-        [ret setObject:md forKey:[[md path] lastPathComponent]];
-    }
-    return ret;
-}
-- (NSArray *)pathsWithPrefix:(NSString *)thePrefix delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base pathsWithPrefix:thePrefix delegate:theDelegate error:error];
-}
-- (BOOL)deleteObjectsForComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base deleteObjectsForComputerUUID:theComputerUUID delegate:theDelegate error:error];
-}
-- (BOOL)deletePaths:(NSArray *)thePaths delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base deletePaths:thePaths delegate:theDelegate error:error];
-}
-- (NSNumber *)fileExistsAtPath:(NSString *)thePath dataSize:(unsigned long long *)theDataSize delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base fileExistsAtPath:thePath dataSize:theDataSize delegate:theDelegate error:error];
-}
-- (NSData *)contentsOfFileAtPath:(NSString *)thePath delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base contentsOfFileAtPath:thePath delegate:theDelegate error:error];
-}
-- (BOOL)writeData:(NSData *)theData toFileAtPath:(NSString *)thePath dataTransferDelegate:(id <DataTransferDelegate>)theDataTransferDelegate targetConnectionDelegate:(id<TargetConnectionDelegate>)theTargetConnectionDelegate error:(NSError **)error {
-    return [base writeData:theData toFileAtPath:thePath dataTransferDelegate:theDataTransferDelegate targetConnectionDelegate:theTargetConnectionDelegate error:error];
-}
-- (BOOL)removeItemAtPath:(NSString *)thePath delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base removeItemAtPath:thePath delegate:theDelegate error:error];
-}
-- (NSNumber *)sizeOfItemAtPath:(NSString *)thePath delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base sizeOfItemAtPath:thePath delegate:theDelegate error:error];
-}
-- (NSNumber *)isObjectRestoredAtPath:(NSString *)thePath delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base isObjectRestoredAtPath:thePath delegate:theDelegate error:error];
-}
-- (BOOL)restoreObjectAtPath:(NSString *)thePath forDays:(NSUInteger)theDays alreadyRestoredOrRestoring:(BOOL *)alreadyRestoredOrRestoring delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base restoreObjectAtPath:thePath forDays:theDays alreadyRestoredOrRestoring:alreadyRestoredOrRestoring delegate:theDelegate error:error];
-}
-- (NSData *)saltDataForComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base saltDataForComputerUUID:theComputerUUID delegate:theDelegate error:error];
-}
-- (BOOL)setSaltData:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base setSaltData:theData forComputerUUID:theComputerUUID delegate:theDelegate error:error];
-}
-@end

SFTPTargetConnection.m

@@ -1,155 +0,0 @@
-/*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#import "SFTPTargetConnection.h"
-#import "SFTPRemoteFS.h"
-#import "Target.h"
-#import "BaseTargetConnection.h"
-#import "S3ObjectMetadata.h"
-
-
-@implementation SFTPTargetConnection
-- (id)initWithTarget:(Target *)theTarget {
-    if (self = [super init]) {
-        NSString *pathPrefix = [[theTarget endpoint] path];
-        if ([pathPrefix isEqualToString:@"/"]) {
-            pathPrefix = @"";
-        }
-        sftpRemoteFS = [[SFTPRemoteFS alloc] initWithTarget:theTarget tempDir:[pathPrefix stringByAppendingString:@"/temp"]];
-        base = [[BaseTargetConnection alloc] initWithTarget:theTarget remoteFS:sftpRemoteFS];
-    }
-    return self;
-}
-- (void)dealloc {
-    [sftpRemoteFS release];
-    [base release];
-    [super dealloc];
-}
-
-
-
-#pragma mark TargetConnection
-- (NSArray *)computerUUIDsWithDelegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base computerUUIDsWithDelegate:theDelegate error:error];
-}
-- (NSArray *)bucketUUIDsForComputerUUID:(NSString *)theComputerUUID deleted:(BOOL)deleted delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base bucketUUIDsForComputerUUID:theComputerUUID deleted:deleted delegate:theDelegate error:error];
-}
-- (NSData *)bucketPlistDataForComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base bucketPlistDataForComputerUUID:theComputerUUID bucketUUID:theBucketUUID deleted:deleted delegate:theDelegate error:error];
-}
-- (BOOL)saveBucketPlistData:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base saveBucketPlistData:theData forComputerUUID:theComputerUUID bucketUUID:theBucketUUID deleted:deleted delegate:theDelegate error:error];
-}
-- (BOOL)deleteBucketPlistDataForComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base deleteBucketPlistDataForComputerUUID:theComputerUUID bucketUUID:theBucketUUID deleted:deleted delegate:theDelegate error:error];
-}
-- (NSData *)computerInfoForComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base computerInfoForComputerUUID:theComputerUUID delegate:theDelegate error:error];
-}
-- (BOOL)saveComputerInfo:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base saveComputerInfo:theData forComputerUUID:theComputerUUID delegate:theDelegate error:error];
-}
-- (NSDictionary *)objectsBySHA1ForTargetEndpoint:(NSURL *)theEndpoint isGlacier:(BOOL)theIsGlacier computerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    NSString *endpointPath = [theEndpoint path];
-    if ([endpointPath isEqualToString:@"/"]) {
-        endpointPath = @"";
-    }
-    NSString *prefix = [NSString stringWithFormat:@"%@/%@/objects/", endpointPath, theComputerUUID];
-    NSArray *objects = [base objectsWithPrefix:prefix delegate:theDelegate error:error];
-    if (objects == nil) {
-        return nil;
-    }
-    NSMutableDictionary *ret = [NSMutableDictionary dictionary];
-    for (S3ObjectMetadata *md in objects) {
-        NSArray *pathComponents = [[md path] pathComponents];
-        NSString *lastPathComponent = [pathComponents lastObject];
-        
-        if ([lastPathComponent length] == 40) {
-            [ret setObject:md forKey:lastPathComponent];
-        } else if ([lastPathComponent length] == 38) {
-            NSString *fragment1 = [pathComponents objectAtIndex:([pathComponents count] - 2)];
-            NSString *fragment2 = [pathComponents objectAtIndex:([pathComponents count] - 1)];
-            NSString *theSHA1 = [fragment1 stringByAppendingString:fragment2];
-            [ret setObject:md forKey:theSHA1];
-        } else if ([lastPathComponent length] == 36) {
-            NSString *fragment1 = [pathComponents objectAtIndex:([pathComponents count] - 3)];
-            NSString *fragment2 = [pathComponents objectAtIndex:([pathComponents count] - 2)];
-            NSString *fragment3 = [pathComponents objectAtIndex:([pathComponents count] - 1)];
-            NSString *theSHA1 = [[fragment1 stringByAppendingString:fragment2] stringByAppendingString:fragment3];
-            [ret setObject:md forKey:theSHA1];
-        } else {
-            HSLogWarn(@"unexpected lastpathcomponent %@ of path %@", lastPathComponent, [md path]);
-        }
-    }
-
-    return ret;
-}
-- (NSArray *)pathsWithPrefix:(NSString *)thePrefix delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base pathsWithPrefix:thePrefix delegate:theDelegate error:error];
-}
-- (BOOL)deleteObjectsForComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base deleteObjectsForComputerUUID:theComputerUUID delegate:theDelegate error:error];
-}
-- (BOOL)deletePaths:(NSArray *)thePaths delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base deletePaths:thePaths delegate:theDelegate error:error];
-}
-- (NSNumber *)fileExistsAtPath:(NSString *)thePath dataSize:(unsigned long long *)theDataSize delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base fileExistsAtPath:thePath dataSize:theDataSize delegate:theDelegate error:error];
-}
-- (NSData *)contentsOfFileAtPath:(NSString *)thePath delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base contentsOfFileAtPath:thePath delegate:theDelegate error:error];
-}
-- (BOOL)writeData:(NSData *)theData toFileAtPath:(NSString *)thePath dataTransferDelegate:(id<DataTransferDelegate>)theDTD targetConnectionDelegate:(id<TargetConnectionDelegate>)theTCD error:(NSError **)error {
-    return [base writeData:theData toFileAtPath:thePath dataTransferDelegate:theDTD targetConnectionDelegate:theTCD error:error];
-}
-- (BOOL)removeItemAtPath:(NSString *)thePath delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base removeItemAtPath:thePath delegate:theDelegate error:error];
-}
-- (NSNumber *)sizeOfItemAtPath:(NSString *)thePath delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base sizeOfItemAtPath:thePath delegate:theDelegate error:error];
-}
-- (NSNumber *)isObjectRestoredAtPath:(NSString *)thePath delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base isObjectRestoredAtPath:thePath delegate:theDelegate error:error];
-}
-- (BOOL)restoreObjectAtPath:(NSString *)thePath forDays:(NSUInteger)theDays alreadyRestoredOrRestoring:(BOOL *)alreadyRestoredOrRestoring delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base restoreObjectAtPath:thePath forDays:theDays alreadyRestoredOrRestoring:alreadyRestoredOrRestoring delegate:theDelegate error:error];
-}
-- (NSData *)saltDataForComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base saltDataForComputerUUID:theComputerUUID delegate:theDelegate error:error];
-}
-- (BOOL)setSaltData:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
-    return [base setSaltData:theData forComputerUUID:theComputerUUID delegate:theDelegate error:error];
-}
-
-@end

System.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,15 +30,13 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#import "RemoteFS.h"
-@class Target;
-@class GoogleDrive;
 
 
-@interface GoogleDriveRemoteFS : NSObject <RemoteFS> {
+
-    Target *target;
+@interface System : NSObject {
-    GoogleDrive *googleDrive;
+    
 }
-- (id)initWithTarget:(Target *)theTarget;
++ (NSString *)productVersion:(NSError **)error;
-
++ (NSInteger)productMajorVersion:(NSError **)error;
++ (NSInteger)productMinorVersion:(NSError **)error;
 @end

System.m

@@ -0,0 +1,78 @@
+/*
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
+ 
+ All rights reserved.
+ 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ 
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 
+ * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
+ their contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+#import "System.h"
+#import "RegexKitLite.h"
+
+
+#define SYSTEM_VERSION_PLIST @"/System/Library/CoreServices/SystemVersion.plist"
+
+@implementation System
++ (NSString *)productVersion:(NSError **)error {
+    NSDictionary *dict = [NSDictionary dictionaryWithContentsOfFile:SYSTEM_VERSION_PLIST];
+    if (dict == nil) {
+        SETNSERROR(@"SystemErrorDomain", -1, @"failed to parse %@", SYSTEM_VERSION_PLIST);
+        return nil;
+    }
+    NSString *ret = [dict objectForKey:@"ProductVersion"];
+    if (ret == nil) {
+        SETNSERROR(@"SystemErrorDomain", -1, @"ProductVersion not found in %@", SYSTEM_VERSION_PLIST);
+    }
+    return ret;
+}
++ (NSInteger)productMajorVersion:(NSError **)error {
+    NSString *productVersion = [System productVersion:error];
+    if (productVersion == nil) {
+        return -1;
+    }
+    NSRange range = [productVersion rangeOfRegex:@"^(\\d+)\\.\\d+" capture:1];
+    if (range.location == NSNotFound) {
+        return -1;
+    }
+    NSString *majorVersion = [productVersion substringWithRange:range];
+    return [majorVersion integerValue];
+}
++ (NSInteger)productMinorVersion:(NSError **)error {
+    NSString *productVersion = [System productVersion:error];
+    if (productVersion == nil) {
+        return -1;
+    }
+    NSRange range = [productVersion rangeOfRegex:@"^\\d+\\.(\\d+)" capture:1];
+    if (range.location == NSNotFound) {
+        return -1;
+    }
+    NSString *minorVersion = [productVersion substringWithRange:range];
+    return [minorVersion integerValue];
+}
+@end

Target.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,45 +31,84 @@
  */
 
 
-@protocol TargetConnection;
+
-@class S3Service;
+@class DictNode;
 @class BufferedInputStream;
+@class BufferedOutputStream;
+@class S3Service;
+@class TargetConnection;
 
 
 enum TargetType {
     kTargetAWS = 0,
-    kTargetSFTP = 1,
+    kTargetLocal = 12
-    kTargetGreenQloud = 2,
-    kTargetDreamObjects = 3,
-    kTargetGoogleCloudStorage = 4,
-    kTargetS3Compatible = 5,
-    kTargetGoogleDrive = 6
 };
 typedef int TargetType;
 
 
+
+/*
+ * Example endpoints:
+ * https://AKIAIYUK3N3TME6L4HFA@s3.amazonaws.com/arq-akiaiyuk3n3tme6l4hfa-us-east-1
+ * sftp://stefan@filosync.reitshamer.com/home/stefan
+ */
+
 @interface Target : NSObject {
     NSString *uuid;
+    NSString *nickname;
     NSURL *endpoint;
+    int32_t awsRequestSignatureVersion;
+    NSString *oAuth2ClientId;
+    NSString *oAuth2RedirectURI;
+    
     TargetType targetType;
-    NSString *secret;
-    NSString *passphrase;
-
-    BOOL budgetEnabled;
-    double budgetDollars;
-    uint32_t budgetGB;
-    BOOL useRRS;
 }
 
-- (id)initWithEndpoint:(NSURL *)theEndpoint secret:(NSString *)theSecret passphrase:(NSString *)thePassphrase;
+- (id)initWithUUID:(NSString *)theUUID
+          nickname:(NSString *)theNickname
+          endpoint:(NSURL *)theEndpoint
+awsRequestSignatureVersion:(int32_t)theAWSRequestSignatureVersion;
+
+- (id)initWithPlist:(DictNode *)thePlist;
+
 - (id)initWithBufferedInputStream:(BufferedInputStream *)theBIS error:(NSError **)error;
 
+- (NSString *)errorDomain;
+
 - (NSString *)targetUUID;
+- (NSString *)nickname;
 - (NSURL *)endpoint;
 - (NSString *)endpointDisplayName;
+
+- (int)awsRequestSignatureVersion;
+
 - (NSString *)secret:(NSError **)error;
+- (BOOL)setSecret:(NSString *)theSecret trustedAppPaths:(NSArray *)theTrustedAppPaths error:(NSError **)error;
+- (BOOL)deleteSecret:(NSError **)error;
+
 - (NSString *)passphrase:(NSError **)error;
-- (TargetType)targetType;
+- (BOOL)setPassphrase:(NSString *)theSecret trustedAppPaths:(NSArray *)theTrustedAppPaths error:(NSError **)error;
-- (id <TargetConnection>)newConnection;
+- (BOOL)deletePassphrase:(NSError **)error;
+
+- (NSString *)oAuth2ClientId;
+- (void)setOAuth2ClientId:(NSString *)value;
+
+- (NSString *)oAuth2RedirectURI;
+- (void)setOAuth2RedirectURI:(NSString *)value;
+
+- (NSString *)oAuth2ClientSecret:(NSError **)error;
+- (BOOL)setOAuth2ClientSecret:(NSString *)theSecret trustedAppPaths:(NSArray *)theTrustedAppPaths error:(NSError **)error;
+- (BOOL)deleteOAuth2ClientSecret:(NSError **)error;
+
+- (DictNode *)toPlist;
+
+- (BOOL)writeTo:(BufferedOutputStream *)theBOS error:(NSError **)error;
+- (void)writeTo:(NSData *)data;
+
 - (S3Service *)s3:(NSError **)error;
+
+- (TargetConnection *)newConnection:(NSError **)error;
+
+- (TargetType)targetType;
+- (BOOL)canAccessFilesByPath;
 @end

Target.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,29 +31,73 @@
  */
 
 
+
 #import "Target.h"
-#import "NSString_extra.h"
+#import "DictNode.h"
-#import "AWSRegion.h"
+#import "StringIO.h"
-#import "SFTPTargetConnection.h"
-#import "GoogleDriveTargetConnection.h"
-#import "S3TargetConnection.h"
-#import "S3Service.h"
-#import "S3AuthorizationProvider.h"
-#import "TargetSchedule.h"
 #import "DoubleIO.h"
 #import "BooleanIO.h"
 #import "IntegerIO.h"
-#import "StringIO.h"
+#import "AWSRegion.h"
+#import "RemoteFS.h"
+#import "RegexKitLite.h"
+#import "S3AuthorizationProviderFactory.h"
+#import "S3Service.h"
+#import "AWSRegion.h"
+#import "TargetConnection.h"
+#import "IntegerNode.h"
+#import "KeychainFactory.h"
+#import "KeychainItem.h"
+#import "Keychain.h"
+
+static NSString *ARQ_RESTORE_TARGET_KEYCHAIN_LABEL = @"arq_restore target";
+static NSString *ARQ_RESTORE_PASSPHRASE_KEYCHAIN_LABEL = @"arq_restore passphrase";
+static NSString *ARQ_RESTORE_OAUTH2_CLIENT_SECRET_KEYCHAIN_LABEL = @"arq_restore oauth2 client secret";
 
 
 @implementation Target
-- (id)initWithEndpoint:(NSURL *)theEndpoint secret:(NSString *)theSecret passphrase:(NSString *)thePassphrase {
+- (id)initWithUUID:(NSString *)theUUID
+          nickname:(NSString *)theNickname
+          endpoint:(NSURL *)theEndpoint
+awsRequestSignatureVersion:(int32_t)theAWSRequestSignatureVersion {
     if (self = [super init]) {
-        uuid = [[NSString stringWithRandomUUID] retain];
+        uuid = [theUUID retain];
+        nickname = [theNickname retain];
         endpoint = [theEndpoint retain];
-        secret = [theSecret retain];
+        awsRequestSignatureVersion = theAWSRequestSignatureVersion;
         targetType = [self targetTypeForEndpoint];
-        passphrase = [thePassphrase retain];
+        NSAssert(endpoint != nil, @"endpoint may not be nil");
+    }
+    return self;
+}
+- (id)initWithPlist:(DictNode *)thePlist {
+    if (self = [super init]) {
+        uuid = [[[thePlist stringNodeForKey:@"uuid"] stringValue] copy];
+        nickname = [[[thePlist stringNodeForKey:@"nickname"] stringValue] copy];
+        endpoint = [[NSURL URLWithString:[[thePlist stringNodeForKey:@"endpointDescription"] stringValue]] copy];
+        if ([[endpoint path] hasPrefix:@"//"] && [[endpoint path] length] > 2) {
+            NSString *endpointDescription = [endpoint description];
+            NSString *path = [endpoint path];
+            NSString *prefix = [endpointDescription substringToIndex:[endpointDescription length] - [path length]];
+            NSString *fixedPath = [[endpoint path] substringFromIndex:1];
+            NSString *fixedEndpointDescription = [prefix stringByAppendingString:fixedPath];
+            [endpoint release];
+            endpoint = [[NSURL URLWithString:fixedEndpointDescription] copy];
+        }
+        if ([thePlist containsKey:@"awsRequestSignatureVersion"]) {
+            awsRequestSignatureVersion = (int32_t)[[thePlist integerNodeForKey:@"awsRequestSignatureVersion"] intValue];
+        } else {
+            if ([AWSRegion regionWithS3Endpoint:endpoint] != nil) {
+                awsRequestSignatureVersion = 4;
+            } else {
+                awsRequestSignatureVersion = 2;
+            }
+        }
+        oAuth2ClientId = [[[thePlist stringNodeForKey:@"oAuth2ClientId"] stringValue] copy];
+        oAuth2RedirectURI = [[[thePlist stringNodeForKey:@"oAuth2RedirectURI"] stringValue] copy];
+        
+        targetType = [self targetTypeForEndpoint];
+        NSAssert(endpoint != nil, @"endpoint may not be nil");
     }
     return self;
 }
@@ -64,23 +108,34 @@
             return nil;
         }
         [uuid retain];
+        
+        if (![StringIO read:&nickname from:theBIS error:error]) {
+            [self release];
+            return nil;
+        }
+        [nickname retain];
+
         NSString *theEndpointDescription = nil;
         if (![StringIO read:&theEndpointDescription from:theBIS error:error]) {
             [self release];
             return nil;
         }
-        TargetSchedule *targetSchedule = [[TargetSchedule alloc] initWithBufferedInputStream:theBIS error:error];
+        if (!![IntegerIO readInt32:&awsRequestSignatureVersion from:theBIS error:error]) {
-        if (targetSchedule == nil) {
             [self release];
             return nil;
         }
-        if (![BooleanIO read:&budgetEnabled from:theBIS error:error]
+
-            || ![DoubleIO read:&budgetDollars from:theBIS error:error]
+        if (![StringIO read:&oAuth2ClientId from:theBIS error:error]) {
-            || ![BooleanIO read:&useRRS from:theBIS error:error]
-            || ![IntegerIO readUInt32:&budgetGB from:theBIS error:error]) {
             [self release];
             return nil;
         }
+        [oAuth2ClientId retain];
+        
+        if (![StringIO read:&oAuth2RedirectURI from:theBIS error:error]) {
+            [self release];
+            return nil;
+        }
+        [oAuth2RedirectURI retain];
         
         endpoint = [[NSURL URLWithString:theEndpointDescription] copy];
         targetType = [self targetTypeForEndpoint];
@@ -88,81 +143,199 @@
     }
     return self;
 }
-
 - (void)dealloc {
     [uuid release];
+    [nickname release];
     [endpoint release];
-    [secret release];
+    [oAuth2ClientId release];
-    [passphrase release];
+    [oAuth2RedirectURI release];
     [super dealloc];
 }
 
 - (NSString *)errorDomain {
     return @"TargetErrorDomain";
 }
+
 - (NSString *)targetUUID {
     return uuid;
 }
+- (NSString *)nickname {
+    return nickname;
+}
 - (NSURL *)endpoint {
     return endpoint;
 }
 - (NSString *)endpointDisplayName {
-    TargetType theTargetType = [self targetType];
+    return nickname;
-    switch (theTargetType) {
-        case kTargetAWS:
-            return @"Amazon";
-        case kTargetGreenQloud:
-            return @"GreenQloud";
-        case kTargetDreamObjects:
-            return @"DreamObjects";
-        case kTargetGoogleCloudStorage:
-            return @"Google Cloud Storage";
-        case kTargetGoogleDrive:
-            return @"Google Drive Storage";
-    }
-    return [endpoint host];
 }
+- (int)awsRequestSignatureVersion {
+    return awsRequestSignatureVersion;
+}
+
 - (NSString *)secret:(NSError **)error {
-    return secret;
+    NSError *myError = nil;
-}
+    KeychainItem *item = [[KeychainFactory keychain] existingItemWithLabel:ARQ_RESTORE_TARGET_KEYCHAIN_LABEL account:uuid error:&myError];
-- (NSString *)passphrase:(NSError **)error {
+    if (item == nil) {
-    if (passphrase == nil) {
+        SETERRORFROMMYERROR;
-        SETNSERROR([self errorDomain], ERROR_NOT_FOUND, @"passphrase not given");
+        if ([myError code] == ERROR_NOT_FOUND) {
+            SETNSERROR([self errorDomain], ERROR_MISSING_SECRET, @"Secret key not found in keychain for target %@ (%@)", uuid, endpoint);
+        }
         return nil;
     }
-    return passphrase;
+    return [[[NSString alloc] initWithData:[item passwordData] encoding:NSUTF8StringEncoding] autorelease];
 }
-- (TargetType)targetType {
+- (BOOL)setSecret:(NSString *)theSecret trustedAppPaths:(NSArray *)theTrustedAppPaths error:(NSError **)error {
-    return targetType;
+    NSString *label = ARQ_RESTORE_TARGET_KEYCHAIN_LABEL;
+    [[KeychainFactory keychain] destroyItemForLabel:label account:uuid error:NULL];
+    KeychainItem *item = [[KeychainFactory keychain] createOrUpdateItemWithLabel:label account:uuid passwordData:[theSecret dataUsingEncoding:NSUTF8StringEncoding] trustedAppPaths:theTrustedAppPaths error:error];
+    return item != nil;
 }
-- (id <TargetConnection>)newConnection {
+- (BOOL)deleteSecret:(NSError **)error {
-    id <TargetConnection> ret = nil;
+    return [[KeychainFactory keychain] destroyItemForLabel:ARQ_RESTORE_TARGET_KEYCHAIN_LABEL account:uuid error:error];
-    if (targetType == kTargetSFTP) {
+}
-        ret = [[SFTPTargetConnection alloc] initWithTarget:self];
+
-    } else if (targetType == kTargetGoogleDrive) {
+- (NSString *)passphrase:(NSError **)error {
-        ret = [[GoogleDriveTargetConnection alloc] initWithTarget:self];
+    KeychainItem *item = [[KeychainFactory keychain] existingItemWithLabel:ARQ_RESTORE_PASSPHRASE_KEYCHAIN_LABEL account:uuid error:error];
-    } else {
+    if (item == nil) {
-        ret = [[S3TargetConnection alloc] initWithTarget:self];
+        return nil;
     }
+    return [[[NSString alloc] initWithData:[item passwordData] encoding:NSUTF8StringEncoding] autorelease];
+}
+- (BOOL)setPassphrase:(NSString *)theSecret trustedAppPaths:(NSArray *)theTrustedAppPaths error:(NSError **)error {
+    NSString *label = ARQ_RESTORE_PASSPHRASE_KEYCHAIN_LABEL;
+    [[KeychainFactory keychain] destroyItemForLabel:label account:uuid error:NULL];
+    KeychainItem *item = [[KeychainFactory keychain] createOrUpdateItemWithLabel:label account:uuid passwordData:[theSecret dataUsingEncoding:NSUTF8StringEncoding] trustedAppPaths:theTrustedAppPaths error:error];
+    return item != nil;
+}
+- (BOOL)deletePassphrase:(NSError **)error {
+    return [[KeychainFactory keychain] destroyItemForLabel:ARQ_RESTORE_PASSPHRASE_KEYCHAIN_LABEL account:uuid error:error];
+}
+
+- (NSString *)oAuth2ClientId {
+    return oAuth2ClientId;
+}
+- (void)setOAuth2ClientId:(NSString *)value {
+    [value retain];
+    [oAuth2ClientId release];
+    oAuth2ClientId = value;
+}
+
+- (NSString *)oAuth2RedirectURI {
+    return oAuth2RedirectURI;
+}
+- (void)setOAuth2RedirectURI:(NSString *)value {
+    [value retain];
+    [oAuth2RedirectURI release];
+    oAuth2RedirectURI = value;
+}
+
+- (NSString *)oAuth2ClientSecret:(NSError **)error {
+    NSError *myError = nil;
+    KeychainItem *item = [[KeychainFactory keychain] existingItemWithLabel:ARQ_RESTORE_OAUTH2_CLIENT_SECRET_KEYCHAIN_LABEL account:uuid error:&myError];
+    if (item == nil) {
+        SETERRORFROMMYERROR;
+        if ([myError code] == ERROR_NOT_FOUND) {
+            SETNSERROR([self errorDomain], ERROR_MISSING_SECRET, @"Client secret not found in keychain for target %@ (%@)", uuid, endpoint);
+        }
+        return nil;
+    }
+    return [[[NSString alloc] initWithData:[item passwordData] encoding:NSUTF8StringEncoding] autorelease];
+}
+- (BOOL)setOAuth2ClientSecret:(NSString *)theSecret trustedAppPaths:(NSArray *)theTrustedAppPaths error:(NSError **)error {
+    NSString *label = ARQ_RESTORE_OAUTH2_CLIENT_SECRET_KEYCHAIN_LABEL;
+    [[KeychainFactory keychain] destroyItemForLabel:label account:uuid error:NULL];
+    KeychainItem *item = [[KeychainFactory keychain] createOrUpdateItemWithLabel:label account:uuid passwordData:[theSecret dataUsingEncoding:NSUTF8StringEncoding] trustedAppPaths:theTrustedAppPaths error:error];
+    return item != nil;
+}
+- (BOOL)deleteOAuth2ClientSecret:(NSError **)error {
+    return [[KeychainFactory keychain] destroyItemForLabel:ARQ_RESTORE_OAUTH2_CLIENT_SECRET_KEYCHAIN_LABEL account:uuid error:error];
+}
+
+
+- (DictNode *)toPlist {
+    DictNode *ret = [[[DictNode alloc] init] autorelease];
+    [ret putString:@"s3" forKey:@"targetType"]; // Used by TargetFactory
+    [ret putString:nickname forKey:@"nickname"];
+    [ret putString:uuid forKey:@"uuid"];
+    [ret putString:[endpoint absoluteString] forKey:@"endpointDescription"];
+    [ret putInt:awsRequestSignatureVersion forKey:@"awsRequestSignatureVersion"];
+    [ret putString:oAuth2ClientId forKey:@"oAuth2ClientId"];
+    [ret putString:oAuth2RedirectURI forKey:@"oAuth2RedirectURI"];
     return ret;
 }
+
+- (BOOL)writeTo:(BufferedOutputStream *)theBOS error:(NSError **)error {
+    return [StringIO write:uuid to:theBOS error:error]
+    && [StringIO write:nickname to:theBOS error:error]
+    && [StringIO write:[endpoint description] to:theBOS error:error]
+    && [IntegerIO writeInt32:awsRequestSignatureVersion to:theBOS error:error]
+    && [StringIO write:oAuth2ClientId to:theBOS error:error]
+    && [StringIO write:oAuth2RedirectURI to:theBOS error:error];
+}
+- (void)writeTo:(NSMutableData *)data {
+    [StringIO write:uuid to:data];
+    [StringIO write:nickname to:data];
+    [StringIO write:[endpoint description] to:data];
+    [IntegerIO writeInt32:awsRequestSignatureVersion to:data];
+    [StringIO write:oAuth2ClientId to:data];
+    [StringIO write:oAuth2RedirectURI to:data];
+}
+
 - (S3Service *)s3:(NSError **)error {
-    if ([self targetType] == kTargetSFTP || [self targetType] == kTargetGoogleDrive) {
+    if ([self targetType] == kTargetLocal) {
         SETNSERROR([self errorDomain], -1, @"cannot create S3Service for endpoint %@", endpoint);
         return nil;
     }
+    AWSRegion *awsRegion = [AWSRegion regionWithS3Endpoint:endpoint];
+    if (awsRegion == nil) {
+        // Default to us-east-1 for non-AWS endpoints.
+        awsRegion = [AWSRegion usEast1];
+    }
+    NSString *secret = [self secret:error];
+    if (secret == nil) {
+        return nil;
+    }
     
-    S3AuthorizationProvider *sap = [[[S3AuthorizationProvider alloc] initWithAccessKey:[endpoint user] secretKey:secret] autorelease];
+    id <S3AuthorizationProvider> sap = [[S3AuthorizationProviderFactory sharedS3AuthorizationProviderFactory] providerForEndpoint:endpoint
+                                                                                                                        accessKey:[endpoint user]
+                                                                                                                        secretKey:secret
+                                                                                                                 signatureVersion:awsRequestSignatureVersion
+                                                                                                                        awsRegion:awsRegion];
     NSString *portString = @"";
     if ([[endpoint port] intValue] != 0) {
         portString = [NSString stringWithFormat:@":%d", [[endpoint port] intValue]];
     }
     NSURL *s3Endpoint = [NSURL URLWithString:[NSString stringWithFormat:@"%@://%@%@", [endpoint scheme], [endpoint host], portString]];
-    return [[[S3Service alloc] initWithS3AuthorizationProvider:sap endpoint:s3Endpoint useAmazonRRS:NO] autorelease];
+    return [[[S3Service alloc] initWithS3AuthorizationProvider:sap endpoint:s3Endpoint] autorelease];
+}
+
+- (TargetConnection *)newConnection:(NSError **)error {
+    return [[TargetConnection alloc] initWithTarget:self];
+}
+- (TargetType)targetType {
+    return targetType;
+}
+- (BOOL)canAccessFilesByPath {
+    return YES;
 }
 
 
 #pragma mark NSObject
+- (BOOL)isEqual:(id)other {
+    if (other == self) {
+        return YES;
+    }
+    if (other == nil || ![other isKindOfClass:[self class]]) {
+        return NO;
+    }
+    Target *target = (Target *)other;
+    DictNode *myPlist = [self toPlist];
+    DictNode *otherPlist = [target toPlist];
+    return [myPlist isEqualToDictNode:otherPlist];
+}
+- (NSUInteger)hash {
+    return [uuid hash];
+}
 - (NSString *)description {
     return [NSString stringWithFormat:@"%@:%@", [endpoint host], [endpoint path]];
 }
@@ -170,25 +343,10 @@
 
 #pragma mark internal
 - (TargetType)targetTypeForEndpoint {
-    if ([[[self endpoint] scheme] isEqualToString:@"sftp"]) {
+    if ([[[self endpoint] scheme] isEqualToString:@"file"]) {
-        return kTargetSFTP;
+        return kTargetLocal;
     }
-    if ([[[self endpoint] scheme] isEqualToString:@"googledrive"]) {
+    
-        return kTargetGoogleDrive;
+    return kTargetAWS;
-    }
-    AWSRegion *awsRegion = [AWSRegion regionWithS3Endpoint:[self endpoint]];
-    if (awsRegion != nil) {
-        return kTargetAWS;
-    }
-    if ([[[self endpoint] host] isEqualToString:@"w.greenqloud.com"]) {
-        return kTargetGreenQloud;
-    }
-    if ([[[self endpoint] host] isEqualToString:@"objects.dreamhost.com"]) {
-        return kTargetDreamObjects;
-    }
-    if ([[[self endpoint] host] isEqualToString:@"storage.googleapis.com"]) {
-        return kTargetGoogleCloudStorage;
-    }
-    return kTargetS3Compatible;
 }
 @end

TargetConnection.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,13 +31,31 @@
  */
 
 
+
+@class Target;
+#import "RemoteFS.h"
+@class Item;
 @protocol DataTransferDelegate;
+@protocol DeleteDelegate;
+
 
 @protocol TargetConnectionDelegate <NSObject>
 - (BOOL)targetConnectionShouldRetryOnTransientError:(NSError **)error;
 @end
 
-@protocol TargetConnection <NSObject>
+
+@interface TargetConnection : NSObject {
+    Target *target;
+    NSString *pathPrefix;
+    NSMutableDictionary *remoteFSByThreadId;
+    NSLock *lock;
+}
+- (id)initWithTarget:(Target *)theTarget;
+
+- (BOOL)updateFingerprintWithTargetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error;
+
+- (Item *)itemAtPath:(NSString *)thePath targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error;
+- (NSDictionary *)itemsByNameAtPath:(NSString *)thePath targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error;
 
 - (NSArray *)computerUUIDsWithDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
 - (NSArray *)bucketUUIDsForComputerUUID:(NSString *)theComputerUUID deleted:(BOOL)deleted delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
@@ -49,20 +67,34 @@
 - (NSData *)computerInfoForComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
 - (BOOL)saveComputerInfo:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
 
-- (NSDictionary *)objectsBySHA1ForTargetEndpoint:(NSURL *)theEndpoint isGlacier:(BOOL)theIsGlacier computerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (NSArray *)pathsWithPrefix:(NSString *)thePrefix delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
 - (BOOL)deleteObjectsForComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (BOOL)deletePaths:(NSArray *)thePaths delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
 
 - (NSNumber *)fileExistsAtPath:(NSString *)thePath dataSize:(unsigned long long *)theDataSize delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
 - (NSData *)contentsOfFileAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
+- (NSData *)contentsOfRange:(NSRange)theRange ofFileAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
 - (BOOL)writeData:(NSData *)theData toFileAtPath:(NSString *)thePath dataTransferDelegate:(id <DataTransferDelegate>)theDelegate targetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
 - (BOOL)removeItemAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (NSNumber *)sizeOfItemAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
+
+- (NSString *)checksumOfFileAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
+
+- (NSNumber *)aggregateSizeOfDirectoryAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
 - (NSNumber *)isObjectRestoredAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
-- (BOOL)restoreObjectAtPath:(NSString *)thePath forDays:(NSUInteger)theDays alreadyRestoredOrRestoring:(BOOL *)alreadyRestoredOrRestoring delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
+- (BOOL)restoreObjectAtPath:(NSString *)thePath forDays:(NSUInteger)theDays tier:(int)theGlacierRetrievalTier alreadyRestoredOrRestoring:(BOOL *)alreadyRestoredOrRestoring delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
 
 - (NSData *)saltDataForComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
 - (BOOL)setSaltData:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
+- (BOOL)deleteSaltDataForComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
 
+- (NSData *)encryptionDataForComputerUUID:(NSString *)theComputerUUID encryptionVersion:(int)theEncryptionVersion delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
+- (BOOL)setEncryptionData:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID encryptionVersion:(int)theEncryptionVersion delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error;
+
+- (NSDictionary *)pathsBySHA1WithIsGlacier:(BOOL)theIsGlacier computerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error;
+
+- (NSNumber *)freeBytesAtPath:(NSString *)thePath targetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
+
+- (BOOL)clearCachedItemsForDirectory:(NSString *)theDirectory error:(NSError **)error;
+- (BOOL)clearAllCachedData:(NSError **)error;
+
+- (NSNumber *)chunkerVersionForComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
+- (BOOL)setChunkerVersion:(NSInteger)theChunkerVersion forComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error;
 @end

TargetConnection.m

@@ -0,0 +1,550 @@
+/*
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
+ 
+ All rights reserved.
+ 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ 
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 
+ * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
+ their contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+#import "TargetConnection.h"
+#import "Target.h"
+#import "RegexKitLite.h"
+#import "S3ObjectMetadata.h"
+#import "Item.h"
+#import "RemoteFS.h"
+#import "S3AuthorizationProviderFactory.h"
+#import "S3Service.h"
+#import "LocalItemFS.h"
+#import "UserLibrary_Arq.h"
+#import "AWSRegion.h"
+#import "MD5Hash.h"
+#import "IntegerIO.h"
+#import "DataInputStream.h"
+#import "BufferedInputStream.h"
+
+
+@implementation TargetConnection
+- (id)initWithTarget:(Target *)theTarget {
+    if (self = [super init]) {
+        target = [theTarget retain];
+        if ([[[theTarget endpoint] path] isEqualToString:@"/"]) {
+            pathPrefix = [@"" retain];
+        } else {
+            pathPrefix = [[[theTarget endpoint] path] retain];
+        }
+        
+        remoteFSByThreadId = [[NSMutableDictionary alloc] init];
+        lock = [[NSLock alloc] init];
+        [lock setName:@"TargetConnection lock"];
+    }
+    return self;
+}
+
+- (void)dealloc {
+    [target release];
+    [pathPrefix release];
+    [remoteFSByThreadId release];
+    [lock release];
+    [super dealloc];
+}
+
+- (BOOL)updateFingerprintWithTargetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return NO;
+    }
+    return [remoteFS updateFingerprintWithTargetConnectionDelegate:theTCD error:error];
+}
+
+- (Item *)itemAtPath:(NSString *)thePath targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error {
+    return [[self remoteFS:error] itemAtPath:thePath targetConnectionDelegate:theTCD error:error];
+}
+- (NSDictionary *)itemsByNameAtPath:(NSString *)thePath targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error {
+    return [[self remoteFS:error] itemsByNameInDirectory:thePath targetConnectionDelegate:theTCD error:error];
+}
+
+- (NSArray *)computerUUIDsWithDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return nil;
+    }
+    
+    // Don't clear the cache because it means you'd have to query for the entire list of objects if you try to restore a file.
+    // Instead, tell the user to use the Clear Cache button.
+//    if (![remoteFS clearCacheForPath:[[target endpoint] path] error:error]) {
+//        return nil;
+//    }
+    NSDictionary *itemsByName = [remoteFS itemsByNameInDirectory:[[target endpoint] path] useCachedData:NO targetConnectionDelegate:theDelegate error:error];
+    if (itemsByName == nil) {
+        return nil;
+    }
+    
+    HSLogDebug(@"found %ld items at %@: %@", [itemsByName count], [target endpoint], [itemsByName allKeys]);
+    
+    NSMutableArray *ret = [NSMutableArray array];
+    for (Item *item in [itemsByName allValues]) {
+        if ([item.name rangeOfRegex:@"^(\\S{8}-\\S{4}-\\S{4}-\\S{4}-\\S{12})$"].location != NSNotFound) {
+            [ret addObject:item.name];
+        } else {
+            HSLogDebug(@"%@ is not a UUID; skipping", item.name);
+        }
+    }
+    HSLogDebug(@"found %ld UUIDs at %@: %@", [ret count], [target endpoint], ret);
+    return ret;
+}
+
+- (NSArray *)bucketUUIDsForComputerUUID:(NSString *)theComputerUUID deleted:(BOOL)deleted delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return nil;
+    }
+    NSString *subdir = deleted ? @"deletedbuckets" : @"buckets";
+    NSString *bucketsDir = [NSString stringWithFormat:@"%@/%@/%@", pathPrefix, theComputerUUID, subdir];
+    NSDictionary *itemsByName = [remoteFS itemsByNameInDirectory:bucketsDir useCachedData:NO targetConnectionDelegate:theDelegate error:error];
+    if (itemsByName == nil) {
+        return nil;
+    }
+    
+    NSMutableArray *ret = [NSMutableArray array];
+    for (Item *item in [itemsByName allValues]) {
+        if ([item.name rangeOfRegex:@"^(\\S{8}-\\S{4}-\\S{4}-\\S{4}-\\S{12})$"].location != NSNotFound) {
+            [ret addObject:item.name];
+        }
+    }
+    return ret;
+}
+
+- (NSData *)bucketPlistDataForComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    NSString *subdir = deleted ? @"deletedbuckets" : @"buckets";
+    NSString *path = [NSString stringWithFormat:@"%@/%@/%@/%@", pathPrefix, theComputerUUID, subdir, theBucketUUID];
+    return [[self remoteFS:error] contentsOfFileAtPath:path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
+}
+- (BOOL)saveBucketPlistData:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return NO;
+    }
+    NSString *subdir = deleted ? @"deletedbuckets" : @"buckets";
+    NSString *path = [NSString stringWithFormat:@"%@/%@/%@/%@", pathPrefix, theComputerUUID, subdir, theBucketUUID];
+    NSError *myError = nil;
+    NSData *existingPlist = [remoteFS contentsOfFileAtPath:path dataTransferDelegate:nil targetConnectionDelegate:nil error:&myError];
+    if (existingPlist != nil) {
+        // Copy existing file to /<prefix>/<computerUUID>/bucketdata/<uuid>/plist_history/<timeinterval>/<bucketUUID>
+        NSString *backupPath = [NSString stringWithFormat:@"%@/%@/bucketdata/%@/plist_history/%0.0f/%@", pathPrefix, theComputerUUID, theBucketUUID, [NSDate timeIntervalSinceReferenceDate], theBucketUUID];
+        if (![remoteFS createFileAtomicallyWithData:existingPlist atPath:backupPath dataTransferDelegate:nil targetConnectionDelegate:nil error:&myError]) {
+            HSLogError(@"failed to save previous plist to %@: %@", backupPath, myError);
+        }
+    }
+    Item *item = [remoteFS createFileAtomicallyWithData:theData atPath:path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
+    return item != nil;
+}
+- (BOOL)deleteBucketPlistDataForComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID deleted:(BOOL)deleted delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return NO;
+    }
+    NSString *subdir = deleted ? @"deletedbuckets" : @"buckets";
+    NSString *path = [NSString stringWithFormat:@"%@/%@/%@/%@", pathPrefix, theComputerUUID, subdir, theBucketUUID];
+    return [remoteFS removeItemAtPath:path targetConnectionDelegate:theDelegate error:error];
+}
+
+- (NSData *)computerInfoForComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    NSString *path = [NSString stringWithFormat:@"%@/%@/computerinfo", pathPrefix, theComputerUUID];
+    return [[self remoteFS:error] contentsOfFileAtPath:path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
+}
+- (BOOL)saveComputerInfo:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    NSString *path = [NSString stringWithFormat:@"%@/%@/computerinfo", pathPrefix, theComputerUUID];
+    Item *item = [[self remoteFS:error] createFileAtomicallyWithData:theData atPath:path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
+    return item != nil;
+}
+- (BOOL)deleteObjectsForComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return NO;
+    }
+    return [remoteFS removeItemAtPath:[NSString stringWithFormat:@"%@/%@", pathPrefix, theComputerUUID] targetConnectionDelegate:theDelegate error:error];
+}
+- (NSNumber *)fileExistsAtPath:(NSString *)thePath dataSize:(unsigned long long *)theDataSize delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    NSError *myError = nil;
+    Item *item = [self itemAtPath:thePath targetConnectionDelegate:theDelegate error:&myError];
+    if (item == nil) {
+        if ([myError code] != ERROR_NOT_FOUND) {
+            SETERRORFROMMYERROR;
+            return nil;
+        }
+        return [NSNumber numberWithBool:NO];
+    }
+    if (theDataSize != NULL) {
+        if (!item.isDirectory) {
+            *theDataSize = item.fileSize;
+        }
+    }
+    return [NSNumber numberWithBool:YES];
+}
+- (NSData *)contentsOfFileAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    return [self contentsOfRange:NSMakeRange(NSNotFound, 0) ofFileAtPath:thePath delegate:theDelegate error:error];
+}
+- (NSData *)contentsOfRange:(NSRange)theRange ofFileAtPath:(NSString *)thePath delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    return [[self remoteFS:error] contentsOfRange:theRange ofFileAtPath:thePath dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
+}
+- (BOOL)writeData:(NSData *)theData toFileAtPath:(NSString *)thePath dataTransferDelegate:(id <DataTransferDelegate>)theDataTransferDelegate targetConnectionDelegate:(id <TargetConnectionDelegate>)theTargetConnectionDelegate error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return NO;
+    }
+    Item *item = [remoteFS createFileAtomicallyWithData:theData atPath:thePath dataTransferDelegate:theDataTransferDelegate targetConnectionDelegate:theTargetConnectionDelegate error:error];
+    return item != nil;
+}
+- (BOOL)removeItemAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return NO;
+    }
+    return [remoteFS removeItemAtPath:thePath targetConnectionDelegate:theDelegate error:error];
+}
+
+- (NSString *)checksumOfFileAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return nil;
+    }
+    Item *item = [remoteFS itemAtPath:thePath targetConnectionDelegate:theTCD error:error];
+    if (item == nil) {
+        return nil;
+    }
+    if (item.checksum != nil) {
+        return item.checksum;
+    }
+    
+    HSLogDebug(@"no checksum in Item; downloading %@ to calculate md5 hash", thePath);
+    NSData *data = [self contentsOfFileAtPath:thePath delegate:theTCD error:error];
+    if (data == nil) {
+        return nil;
+    }
+    NSString *md5 = [MD5Hash hashData:data];
+    return [@"md5:" stringByAppendingString:md5];
+}
+
+- (NSNumber *)aggregateSizeOfItemAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return nil;
+    }
+    NSError *myError = nil;
+    Item *item = [remoteFS itemAtPath:thePath targetConnectionDelegate:theTCD error:&myError];
+    if (item == nil) {
+        if ([myError code] != ERROR_NOT_FOUND) {
+            SETERRORFROMMYERROR;
+            return nil;
+        }
+        return [NSNumber numberWithLong:0];
+    }
+    
+    return [self aggregateSizeOfItem:item atPath:thePath delegate:theTCD error:error];
+}
+- (NSNumber *)aggregateSizeOfItem:(Item *)theItem atPath:(NSString *)thePath delegate:(id<TargetConnectionDelegate>)theTCD error:(NSError **)error {
+    unsigned long long total = 0;
+    
+    if (theItem.isDirectory) {
+        NSNumber *childSize = [self aggregateSizeOfDirectoryAtPath:thePath delegate:theTCD error:error];
+        if (childSize == nil) {
+            return nil;
+        }
+        total += [childSize unsignedLongLongValue];
+    } else {
+        total = theItem.fileSize;
+    }
+    return [NSNumber numberWithUnsignedLongLong:total];
+}
+- (NSNumber *)aggregateSizeOfDirectoryAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return nil;
+    }
+    NSDictionary *itemsByName = [remoteFS itemsByNameInDirectory:thePath targetConnectionDelegate:theTCD error:error];
+    if (itemsByName == nil) {
+        return nil;
+    }
+    unsigned long long total = 0;
+    BOOL ret = YES;
+    NSAutoreleasePool *pool = nil;
+    for (Item *item in [itemsByName allValues]) {
+        [pool drain];
+        pool = [[NSAutoreleasePool alloc] init];
+        if (theTCD != nil && ![theTCD targetConnectionShouldRetryOnTransientError:error]) {
+            ret = NO;
+            break;
+        }
+        if (item.isDirectory) {
+            NSString *childPath = [thePath stringByAppendingPathComponent:item.name];
+            NSNumber *size = [self aggregateSizeOfItemAtPath:childPath delegate:theTCD error:error];
+            if (size == nil) {
+                ret = NO;
+                break;
+            }
+            total += [size unsignedLongLongValue];
+        } else {
+            total += item.fileSize;
+        }
+    }
+    if (!ret && error != NULL) {
+        [*error retain];
+    }
+    [pool drain];
+    if (!ret && error != NULL) {
+        [*error autorelease];
+    }
+    if (!ret) {
+        return nil;
+    }
+    return [NSNumber numberWithUnsignedLongLong:total];
+}
+
+- (NSNumber *)isObjectRestoredAtPath:(NSString *)thePath delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    return [[self remoteFS:error] isObjectRestoredAtPath:thePath targetConnectionDelegate:theDelegate error:error];
+}
+- (BOOL)restoreObjectAtPath:(NSString *)thePath forDays:(NSUInteger)theDays tier:(int)theGlacierRetrievalTier alreadyRestoredOrRestoring:(BOOL *)alreadyRestoredOrRestoring delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return NO;
+    }
+    return [remoteFS restoreObjectAtPath:thePath forDays:theDays tier:theGlacierRetrievalTier alreadyRestoredOrRestoring:alreadyRestoredOrRestoring targetConnectionDelegate:theDelegate error:error];
+}
+
+- (NSData *)saltDataForComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    NSString *s3Path = [NSString stringWithFormat:@"%@/%@/salt", pathPrefix, theComputerUUID];
+    return [[self remoteFS:error] contentsOfFileAtPath:s3Path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
+}
+- (BOOL)setSaltData:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return NO;
+    }
+    NSString *s3Path = [NSString stringWithFormat:@"%@/%@/salt", pathPrefix, theComputerUUID];
+    Item *item = [remoteFS createFileAtomicallyWithData:theData atPath:s3Path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
+    return item != nil;
+}
+- (BOOL)deleteSaltDataForComputerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return NO;
+    }
+    NSString *s3Path = [NSString stringWithFormat:@"%@/%@/salt", pathPrefix, theComputerUUID];
+    return [remoteFS removeItemAtPath:s3Path targetConnectionDelegate:theDelegate error:error];
+}
+
+- (NSData *)encryptionDataForComputerUUID:(NSString *)theComputerUUID encryptionVersion:(int)theEncryptionVersion delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    NSString *s3Path = [NSString stringWithFormat:@"%@/%@/encryptionv%d.dat", pathPrefix, theComputerUUID, theEncryptionVersion];
+    return [[self remoteFS:error] contentsOfFileAtPath:s3Path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
+}
+- (BOOL)setEncryptionData:(NSData *)theData forComputerUUID:(NSString *)theComputerUUID encryptionVersion:(int)theEncryptionVersion delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return NO;
+    }
+    NSString *s3Path = [NSString stringWithFormat:@"%@/%@/encryptionv%d.dat", pathPrefix, theComputerUUID, theEncryptionVersion];
+    Item *item = [remoteFS createFileAtomicallyWithData:theData atPath:s3Path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
+    return item != nil;
+}
+
+- (NSDictionary *)pathsBySHA1WithIsGlacier:(BOOL)theIsGlacier computerUUID:(NSString *)theComputerUUID delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return nil;
+    }
+    NSString *objectsDir = [NSString stringWithFormat:@"%@/%@%@/objects", [[target endpoint] path], (theIsGlacier ? @"glacier/" : @""), theComputerUUID];
+    
+    NSMutableDictionary *ret = [NSMutableDictionary dictionary];
+    if (![self addPathsBySHA1InDirectory:objectsDir fromRemoteFS:remoteFS toDictionary:ret delegate:theDelegate error:error]) {
+        return nil;
+    }
+    return ret;
+}
+- (BOOL)addPathsBySHA1InDirectory:(NSString *)theDirectory fromRemoteFS:(RemoteFS *)remoteFS toDictionary:(NSMutableDictionary *)ret delegate:(id<TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    NSDictionary *itemsByName = [remoteFS itemsByNameInDirectory:theDirectory targetConnectionDelegate:theDelegate error:error];
+    if (itemsByName == nil) {
+        return NO;
+    }
+    for (Item *item in [itemsByName allValues]) {
+        if ([item.name length] == 2) {
+            NSString *subdir = [theDirectory stringByAppendingPathComponent:item.name];
+            NSDictionary *subdirItemsByName = [remoteFS itemsByNameInDirectory:subdir targetConnectionDelegate:theDelegate error:error];
+            if (subdirItemsByName == nil) {
+                return NO;
+            }
+            for (Item *subdirItem in [subdirItemsByName allValues]) {
+                NSString *childPath = [subdir stringByAppendingPathComponent:subdirItem.name];
+                if ([subdirItem.name length] == 2) {
+                    NSDictionary *subsubdirItemsByName = [remoteFS itemsByNameInDirectory:childPath targetConnectionDelegate:theDelegate error:error];
+                    if (subsubdirItemsByName == nil) {
+                        return NO;
+                    }
+                    for (NSString *subsubDirName in [subsubdirItemsByName allKeys]) {
+                        NSString *subsubDirPath = [childPath stringByAppendingPathComponent:subsubDirName];
+                        NSString *sha1 = [NSString stringWithFormat:@"%@%@%@", item.name, subdirItem.name, subsubDirName];
+                        [ret setObject:subsubDirPath forKey:sha1];
+                    }
+                } else if ([subdirItem.name length] == 38) {
+                    NSString *sha1 = [item.name stringByAppendingString:subdirItem.name];
+                    [ret setObject:childPath forKey:sha1];
+                } else {
+                    HSLogWarn(@"unexpected object path %@", childPath);
+                }
+            }
+        } else if ([item.name length] == 40) {
+            [ret setObject:[theDirectory stringByAppendingPathComponent:item.name] forKey:item.name];
+        } else {
+            HSLogWarn(@"ignoring unexpected entry %@/%@", theDirectory, item.name);
+        }
+    }
+    return YES;
+}
+
+- (NSNumber *)freeBytesAtPath:(NSString *)thePath targetConnectionDelegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    return [[self remoteFS:error] freeBytesAtPath:(NSString *)thePath targetConnectionDelegate:theDelegate error:error];
+}
+- (BOOL)clearCachedItemsForDirectory:(NSString *)theDirectory error:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return NO;
+    }
+    return [remoteFS clearCacheForPath:theDirectory error:error];
+}
+- (BOOL)clearAllCachedData:(NSError **)error {
+    RemoteFS *remoteFS = [self remoteFS:error];
+    if (remoteFS == nil) {
+        return NO;
+    }
+    // Delete items databases:
+    if (![remoteFS clearCache:error]) {
+        return NO;
+    }
+    NSString *cacheDir = [[UserLibrary arqCachePath] stringByAppendingPathComponent:[target targetUUID]];
+    
+    NSError *myError = nil;
+    HSLogDetail(@"deleting cached data at %@", cacheDir);
+    if ([[NSFileManager defaultManager] fileExistsAtPath:cacheDir] && ![[NSFileManager defaultManager] removeItemAtPath:cacheDir error:&myError]) {
+        HSLogError(@"failed to delete %@: %@", cacheDir, myError);
+        SETERRORFROMMYERROR;
+        return NO;
+    }
+    
+    return YES;
+}
+- (NSNumber *)chunkerVersionForComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    NSString *path = [NSString stringWithFormat:@"%@/%@/chunker_version.dat", pathPrefix, theComputerUUID];
+    NSData *data = [[self remoteFS:error] contentsOfFileAtPath:path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error];
+    if (data == nil) {
+        return nil;
+    }
+    int32_t theChunkerVersion = 0;
+    DataInputStream *dis = [[[DataInputStream alloc] initWithData:data description:@"chunker version"] autorelease];
+    BufferedInputStream *bis = [[[BufferedInputStream alloc] initWithUnderlyingStream:dis] autorelease];
+    if (![IntegerIO readInt32:&theChunkerVersion from:bis error:error]) {
+        return nil;
+    }
+    return [NSNumber numberWithInteger:(NSInteger)theChunkerVersion];
+}
+- (BOOL)setChunkerVersion:(NSInteger)theChunkerVersion forComputerUUID:(NSString *)theComputerUUID delegate:(id <TargetConnectionDelegate>)theDelegate error:(NSError **)error {
+    NSMutableData *data = [NSMutableData data];
+    [IntegerIO writeInt32:(int32_t)theChunkerVersion to:data];
+    NSString *path = [NSString stringWithFormat:@"%@/%@/chunker_version.dat", pathPrefix, theComputerUUID];
+    if (![[self remoteFS:error] createFileAtomicallyWithData:data atPath:path dataTransferDelegate:nil targetConnectionDelegate:theDelegate error:error]) {
+        return NO;
+    }
+    return YES;
+}
+
+
+#pragma mark internal
+- (RemoteFS *)remoteFS:(NSError **)error {
+    [lock lock];
+    NSNumber *threadId = [NSNumber numberWithUnsignedLongLong:(unsigned long long)pthread_self()];
+    RemoteFS *ret = [remoteFSByThreadId objectForKey:threadId];
+    if (ret == nil) {
+        ret = [self newRemoteFS:error];
+        if (ret != nil) {
+            [remoteFSByThreadId setObject:ret forKey:threadId];
+            [ret release];
+        }
+    }
+    [lock unlock];
+    return ret;
+}
+- (RemoteFS *)newRemoteFS:(NSError **)error {
+    NSString *secret = [target secret:error];
+    if (secret == nil) {
+        return nil;
+    }
+    
+    NSError *myError = nil;
+    NSString *oauth2ClientSecret = [target oAuth2ClientSecret:&myError];
+    if (oauth2ClientSecret == nil && [myError code] != ERROR_MISSING_SECRET) {
+        SETERRORFROMMYERROR;
+        return nil;
+    }
+    
+    id <ItemFS> theItemFS = nil;
+    TargetType targetType = [target targetType];
+
+    if (targetType == kTargetLocal) {
+        theItemFS = [[LocalItemFS alloc] initWithEndpoint:[target endpoint] error:error];
+        if (theItemFS == nil) {
+            return nil;
+        }
+        
+    } else if (targetType == kTargetAWS) {
+        AWSRegion *region = [AWSRegion regionWithS3Endpoint:[target endpoint]];
+        if (region == nil) {
+            region = [AWSRegion usEast1];
+        }
+        id <S3AuthorizationProvider> sap = [[S3AuthorizationProviderFactory sharedS3AuthorizationProviderFactory] providerForEndpoint:[target endpoint]
+                                                                                                                            accessKey:[[target endpoint] user]
+                                                                                                                            secretKey:secret
+                                                                                                                     signatureVersion:[target awsRequestSignatureVersion]
+                                                                                                                            awsRegion:region];
+        NSString *portString = @"";
+        if ([[[target endpoint] port] intValue] != 0) {
+            portString = [NSString stringWithFormat:@":%d", [[[target endpoint] port] intValue]];
+        }
+        NSURL *s3Endpoint = [NSURL URLWithString:[NSString stringWithFormat:@"%@://%@%@", [[target endpoint] scheme], [[target endpoint] host], portString]];
+        theItemFS = [[S3Service alloc] initWithS3AuthorizationProvider:sap endpoint:s3Endpoint];
+        
+    } else {
+        SETNSERROR(@"TargetConnectionErrorDomain", -1, @"unknown target type %d", targetType);
+        return nil;
+    }
+    
+    NSAssert(theItemFS != nil, @"theItemFS may not be nil");
+    
+    RemoteFS *ret = [[RemoteFS alloc] initWithItemFS:theItemFS cacheUUID:[target targetUUID]];
+    [theItemFS release];
+    return ret;
+}
+@end

TargetFactory.h

@@ -0,0 +1,64 @@
+/*
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
+ 
+ All rights reserved.
+ 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ 
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 
+ * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
+ their contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+#import "CWLSynthesizeSingleton.h"
+@class Target;
+
+
+@protocol TargetFactoryChangeListener <NSObject>
+- (void)targetFactoryTargetWasAdded:(Target *)theTarget;
+- (void)targetFactoryTargetWasUpdatedFrom:(Target *)theExisting to:(Target *)theUpdated;
+- (void)targetFactoryTargetWasDeleted:(Target *)theTarget;
+@end
+
+@interface TargetFactory : NSObject {
+    NSMutableSet *changeListeners;
+    FSEventStreamRef streamRef;
+    NSMutableArray *sortedTargets;
+}
+
+CWL_DECLARE_SINGLETON_FOR_CLASS(TargetFactory);
+
+- (BOOL)deleteAllTargets:(NSError **)error;
+- (NSArray *)sortedTargets;
+- (Target *)targetWithNickname:(NSString *)theTargetNickname;
+- (Target *)targetWithUUID:(NSString *)theTargetUUID;
+- (BOOL)saveTarget:(Target *)theTarget error:(NSError **)error;
+- (BOOL)replaceTarget:(Target *)theOldTarget withTarget:(Target *)theNewTarget error:(NSError **)error;
+- (BOOL)deleteTarget:(Target *)theTarget error:(NSError **)error;
+- (void)refresh;
+
+- (void)addChangeListener:(id <TargetFactoryChangeListener>)listener;
+- (void)removeChangeListener:(id <TargetFactoryChangeListener>)listener;
+@end

TargetFactory.m

@@ -0,0 +1,296 @@
+/*
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
+ 
+ All rights reserved.
+ 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ 
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 
+ * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
+ their contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+#import "TargetFactory.h"
+#import "DictNode.h"
+#import "StringNode.h"
+#import "Target.h"
+#import "UserLibrary_Arq.h"
+#import "NSFileManager_extra.h"
+#import "KeychainFactory.h"
+#import "Keychain.h"
+#import "KeychainItem.h"
+#import "Streams.h"
+#import "CacheOwnership.h"
+
+
+@interface TargetFactory ()
+- (void)targetFilesChanged;
+@end
+
+
+static void fsEventStreamCallback(ConstFSEventStreamRef streamRef,
+								  void *clientCallBackInfo,
+								  size_t numEvents,
+								  const char *const eventPaths[],
+								  const FSEventStreamEventFlags eventFlags[],
+								  const FSEventStreamEventId eventIds[]) {
+    TargetFactory *targetFactory = (TargetFactory *)clientCallBackInfo;
+    [targetFactory targetFilesChanged];
+}
+
+
+
+
+@implementation TargetFactory
+CWL_SYNTHESIZE_SINGLETON_FOR_CLASS(TargetFactory);
+
+- (id)init {
+    if (self = [super init]) {
+        changeListeners = [[NSMutableSet alloc] init];
+        [self monitorForChanges];
+
+        sortedTargets = [[NSMutableArray alloc] init];
+        NSError *myError = nil;
+        NSArray *theTargets = [self sortedTargets:&myError];
+        if (theTargets == nil) {
+            HSLogError(@"failed to load targets: %@", myError);
+        } else {
+            [sortedTargets setArray:theTargets];
+        }
+    }
+    return self;
+}
+- (BOOL)deleteAllTargets:(NSError **)error {
+    for (Target *target in sortedTargets) {
+        if (![self doDeleteTarget:target error:error]) {
+            return NO;
+        }
+    }
+    return YES;
+}
+- (NSArray *)sortedTargets {
+    return [NSArray arrayWithArray:sortedTargets];
+}
+- (Target *)targetWithNickname:(NSString *)theTargetNickname {
+    for (Target *target in sortedTargets) {
+        if ([[target nickname] isEqualToString:theTargetNickname]) {
+            return target;
+        }
+    }
+    return nil;
+}
+- (Target *)targetWithUUID:(NSString *)theTargetUUID {
+    for (Target *target in sortedTargets) {
+        if ([[target targetUUID] isEqualToString:theTargetUUID]) {
+            return target;
+        }
+    }
+    return nil;
+}
+- (BOOL)saveTarget:(Target *)theTarget error:(NSError **)error {
+    if (![[NSFileManager defaultManager] createDirectoryAtPath:[self targetsDir] withIntermediateDirectories:YES attributes:nil error:error]) {
+        return NO;
+    }
+    DictNode *plist = [theTarget toPlist];
+    NSData *xmlData = [plist XMLData];
+
+    HSLogDebug(@"writing XML data for target %@ to %@", [theTarget targetUUID], [self pathForTarget:theTarget]);
+    
+    if (![Streams writeData:xmlData atomicallyToFile:[self pathForTarget:theTarget] targetUID:[[CacheOwnership sharedCacheOwnership] uid] targetGID:[[CacheOwnership sharedCacheOwnership] gid] bytesWritten:NULL error:error]) {
+        return NO;
+    }
+    
+    [self refresh];
+    
+    return YES;
+}
+- (BOOL)replaceTarget:(Target *)theOldTarget withTarget:(Target *)theNewTarget error:(NSError **)error {
+    if (![self doDeleteTarget:theOldTarget error:error]) {
+        return NO;
+    }
+    return [self saveTarget:theNewTarget error:error];
+}
+- (BOOL)deleteTarget:(Target *)theTarget error:(NSError **)error {
+    if (![self doDeleteTarget:theTarget error:error]) {
+        return NO;
+    }
+    [self refresh];
+    
+    return YES;
+}
+- (void)refresh {
+    [self targetFilesChanged];
+}
+
+- (void)addChangeListener:(id)listener {
+    [changeListeners addObject:listener];
+}
+- (void)removeChangeListener:(id<TargetFactoryChangeListener>)listener {
+    [changeListeners removeObject:listener];
+}
+
+- (void)monitorForChanges {
+    CFAbsoluteTime latency = 1.0; // seconds
+    FSEventStreamContext context = {
+        0,
+        self,
+        NULL,
+        NULL,
+        NULL
+    };
+    NSArray *pathsToWatch = [NSArray arrayWithObject:[self targetsDir]];
+    streamRef = FSEventStreamCreate(NULL,
+                                    (FSEventStreamCallback)&fsEventStreamCallback,
+                                    &context,
+                                    (CFArrayRef)pathsToWatch,
+                                    kFSEventStreamEventIdSinceNow,
+                                    latency,
+                                    kFSEventStreamCreateFlagNoDefer
+                                    );
+    FSEventStreamScheduleWithRunLoop(streamRef, CFRunLoopGetCurrent(), kCFRunLoopDefaultMode);
+    FSEventStreamStart(streamRef);
+}
+
+
+#pragma mark internal
+
+- (NSString *)errorDomain {
+    return @"TargetFactoryErrorDomain";
+}
+- (BOOL)doDeleteTarget:(Target *)theTarget error:(NSError **)error {
+    HSLogDebug(@"deleting XML data for target %@: %@", [theTarget targetUUID], [self pathForTarget:theTarget]);
+    
+    if (![[NSFileManager defaultManager] removeItemAtPath:[self pathForTarget:theTarget] error:error]) {
+        return NO;
+    }
+    NSError *myError = nil;
+    if (![theTarget deleteSecret:&myError]) {
+        HSLogError(@"error deleting secret for %@: %@", theTarget, myError);
+    }
+    if (![theTarget deletePassphrase:&myError]) {
+        HSLogError(@"error deleting passphrase for %@: %@", theTarget, myError);
+    }
+    if (![theTarget deleteOAuth2ClientSecret:&myError]) {
+        HSLogError(@"error deleting secret for %@: %@", theTarget, myError);
+    }
+    
+    NSString *cachePath = [[UserLibrary arqCachePath] stringByAppendingPathComponent:[theTarget targetUUID]];
+    if (![[NSFileManager defaultManager] removeItemAtPath:cachePath error:&myError]) {
+        HSLogError(@"error deleting cache for target %@: %@", theTarget, myError);
+    }
+    
+    return YES;
+}
+- (NSArray *)sortedTargets:(NSError **)error {
+    NSMutableArray *ret = [NSMutableArray array];
+    if ([[NSFileManager defaultManager] fileExistsAtPath:[self targetsDir]]) {
+        NSArray *targetFileNames = [[NSFileManager defaultManager] contentsOfDirectoryAtPath:[self targetsDir] error:error];
+        if (targetFileNames == nil) {
+            return nil;
+        } else {
+            for (NSString *targetFile in targetFileNames) {
+                NSString *targetPath = [[self targetsDir] stringByAppendingPathComponent:targetFile];
+                NSError *myError = nil;
+                Target *target = [self targetWithPlistPath:targetPath error:&myError];
+                if (target == nil) {
+                    HSLogError(@"failed to parse target file %@: %@", targetFile, myError);
+                } else {
+                    
+                    HSLogDebug(@"loaded target %@ from %@", [target targetUUID], targetPath);
+                    
+                    [ret addObject:target];
+                }
+            }
+        }
+    }
+    NSSortDescriptor *descriptor = [[[NSSortDescriptor alloc] initWithKey:@"targetUUID" ascending:YES] autorelease];
+    [ret sortUsingDescriptors:[NSArray arrayWithObject:descriptor]];
+    return ret;
+}
+- (Target *)targetWithPlistPath:(NSString *)thePath error:(NSError **)error {
+    NSData *data = [[NSData alloc] initWithContentsOfFile:thePath options:0 error:error];
+    if (!data) {
+        return nil;
+    }
+    NSError *myError = nil;
+    DictNode *plist = [DictNode dictNodeWithXMLData:data error:&myError];
+    [data release];
+    if (plist == nil) {
+        SETNSERROR([self errorDomain], -1, @"error parsing %@: %@", thePath, [myError localizedDescription]);
+        return nil;
+    }
+    NSString *targetType = [[plist stringNodeForKey:@"targetType"] stringValue];
+    if ([targetType isEqualToString:@"s3"]) {
+        return [[[Target alloc] initWithPlist:plist] autorelease];
+    }
+    SETNSERROR([self errorDomain], -1, @"unknown target type '%@'", targetType);
+    return nil;
+}
+- (NSString *)targetsDir {
+    return [[[UserLibrary arqUserLibraryPath] stringByAppendingPathComponent:@"config"] stringByAppendingPathComponent:@"targets"];
+}
+- (NSString *)pathForTarget:(Target *)theTarget {
+    return [[self targetsDir] stringByAppendingPathComponent:[NSString stringWithFormat:@"%@.target", [theTarget targetUUID]]];
+}
+- (void)targetFilesChanged {
+    NSError *myError = nil;
+    NSArray *theSortedTargets = [self sortedTargets:&myError];
+    if (theSortedTargets == nil) {
+        HSLogError(@"error reloading targets from files: %@", myError);
+        return;
+    }
+    
+    NSMutableDictionary *existingTargetsByUUID = [NSMutableDictionary dictionary];
+    for (Target *existing in sortedTargets) {
+        [existingTargetsByUUID setObject:existing forKey:[existing targetUUID]];
+    }
+    NSMutableDictionary *updatedTargetsByUUID = [NSMutableDictionary dictionary];
+    for (Target *updated in theSortedTargets) {
+        [updatedTargetsByUUID setObject:updated forKey:[updated targetUUID]];
+    }
+    
+    [sortedTargets setArray:theSortedTargets];
+    
+    for (NSString *targetUUID in [existingTargetsByUUID allKeys]) {
+        Target *existing = [existingTargetsByUUID objectForKey:targetUUID];
+        Target *updated = [updatedTargetsByUUID objectForKey:targetUUID];
+        if (updated == nil) {
+            for (id <TargetFactoryChangeListener> listener in changeListeners) {
+                [listener targetFactoryTargetWasDeleted:existing];
+            }
+        } else if (![existing isEqual:updated]) {
+            for (id <TargetFactoryChangeListener> listener in changeListeners) {
+                [listener targetFactoryTargetWasUpdatedFrom:existing to:updated];
+            }
+        }
+        [updatedTargetsByUUID removeObjectForKey:targetUUID];
+    }
+    for (Target *added in [updatedTargetsByUUID allValues]) {
+        for (id <TargetFactoryChangeListener> listener in changeListeners) {
+            [listener targetFactoryTargetWasAdded:added];
+        }
+    }
+}
+@end

TargetSchedule.m

@@ -1,147 +0,0 @@
-/*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
- their contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#import "TargetSchedule.h"
-#import "DictNode.h"
-#import "IntegerNode.h"
-#import "IntegerIO.h"
-#import "BufferedOutputStream.h"
-#import "BooleanNode.h"
-#import "BooleanIO.h"
-
-
-#define TARGET_DATA_VERSION (1)
-
-
-@implementation TargetSchedule
-- (id)initWithScheduleType:(TargetScheduleType)theType
-             numberOfHours:(int)theNumberOfHours
-          minutesAfterHour:(int)theMinutesAfterHour
-                backupHour:(int)theBackupHour
-budgetEnforcementIntervalHours:(int)theBudgetEnforcementIntervalHours
-         pauseDuringWindow:(BOOL)thePauseDuringWindow
-             pauseFromHour:(NSUInteger)thePauseFromHour
-               pauseToHour:(NSUInteger)thePauseToHour {
-    if (self = [super init]) {
-        type = theType;
-        numberOfHours = theNumberOfHours;
-        minutesAfterHour = theMinutesAfterHour;
-        backupHour = theBackupHour;
-        budgetEnforcementIntervalHours = theBudgetEnforcementIntervalHours;
-        pauseDuringWindow = thePauseDuringWindow;
-        pauseFromHour = (uint32_t)thePauseFromHour;
-        pauseToHour = (uint32_t)thePauseToHour;
-    }
-    return self;
-}
-- (id)initWithPlist:(DictNode *)thePlist {
-    if (self = [super init]) {
-        type = [[thePlist integerNodeForKey:@"type"] intValue];
-        numberOfHours = [[thePlist integerNodeForKey:@"numberOfHours"] intValue];
-        minutesAfterHour = [[thePlist integerNodeForKey:@"minutesAfterHour"] intValue];
-        backupHour = [[thePlist integerNodeForKey:@"backupHour"] intValue];
-        budgetEnforcementIntervalHours = [[thePlist integerNodeForKey:@"budgetEnforcementIntervalHours"] intValue];
-        pauseDuringWindow = [[thePlist booleanNodeForKey:@"pauseDuringWindow"] booleanValue];
-        pauseFromHour = [[thePlist integerNodeForKey:@"pauseFromHour"] intValue];
-        pauseToHour = [[thePlist integerNodeForKey:@"pauseToHour"] intValue];
-    }
-    return self;
-}
-- (id)initWithBufferedInputStream:(BufferedInputStream *)theBIS error:(NSError **)error {
-    if (self = [super init]) {
-        uint32_t version = 0;
-        if (![IntegerIO readUInt32:&version from:theBIS error:error]
-            || ![IntegerIO readUInt32:&type from:theBIS error:error]
-            || ![IntegerIO readUInt32:&numberOfHours from:theBIS error:error]
-            || ![IntegerIO readUInt32:&minutesAfterHour from:theBIS error:error]
-            || ![IntegerIO readUInt32:&backupHour from:theBIS error:error]
-            || ![IntegerIO readUInt32:&budgetEnforcementIntervalHours from:theBIS error:error]
-            || ![BooleanIO read:&pauseDuringWindow from:theBIS error:error]
-            || ![IntegerIO readUInt32:&pauseFromHour from:theBIS error:error]
-            || ![IntegerIO readUInt32:&pauseToHour from:theBIS error:error]) {
-            [self release];
-            return nil;
-        }
-    }
-    return self;
-}
-
-- (TargetScheduleType)type {
-    return type;
-}
-- (uint32_t)numberOfHours {
-    return numberOfHours;
-}
-- (uint32_t)minutesAfterHour {
-    return minutesAfterHour;
-}
-- (uint32_t)backupHour {
-    return backupHour;
-}
-- (uint32_t)budgetEnforcementIntervalHours {
-    return budgetEnforcementIntervalHours;
-}
-- (BOOL)pauseDuringWindow {
-    return pauseDuringWindow;
-}
-- (uint32_t)pauseFromHour {
-    return pauseFromHour;
-}
-- (uint32_t)pauseToHour {
-    return pauseToHour;
-}
-- (DictNode *)toPlist {
-    DictNode *ret = [[[DictNode alloc] init] autorelease];
-    [ret putInt:TARGET_DATA_VERSION forKey:@"dataVersion"];
-    [ret putInt:type forKey:@"type"];
-    [ret putInt:numberOfHours forKey:@"numberOfHours"];
-    [ret putInt:minutesAfterHour forKey:@"minutesAfterHour"];
-    [ret putInt:backupHour forKey:@"backupHour"];
-    [ret putInt:budgetEnforcementIntervalHours forKey:@"budgetEnforcementIntervalHours"];
-    [ret putBoolean:pauseDuringWindow forKey:@"pauseDuringWindow"];
-    [ret putInt:pauseFromHour forKey:@"pauseFromHour"];
-    [ret putInt:pauseToHour forKey:@"pauseToHour"];
-    return ret;
-}
-- (BOOL)writeTo:(BufferedOutputStream *)theBOS error:(NSError **)error {
-    return [IntegerIO writeUInt32:TARGET_DATA_VERSION to:theBOS error:error]
-    && [IntegerIO writeUInt32:type to:theBOS error:error]
-    && [IntegerIO writeUInt32:numberOfHours to:theBOS error:error]
-    && [IntegerIO writeUInt32:minutesAfterHour to:theBOS error:error]
-    && [IntegerIO writeUInt32:backupHour to:theBOS error:error]
-    && [IntegerIO writeUInt32:budgetEnforcementIntervalHours to:theBOS error:error]
-    && [BooleanIO write:pauseDuringWindow to:theBOS error:error]
-    && [IntegerIO writeUInt32:pauseFromHour to:theBOS error:error]
-    && [IntegerIO writeUInt32:pauseToHour to:theBOS error:error];
-}
-@end

UserAndComputer.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -34,6 +34,8 @@
 
 
 
+
+
 @interface UserAndComputer : NSObject {
     NSString *userName;
     NSString *computerName;

UserAndComputer.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #import "UserAndComputer.h"
 #import "Computer.h"
 #import "DictNode.h"

UserLibrary_Arq.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -32,6 +32,7 @@
 
 
 
+
 #import "UserLibrary.h"
 
 @interface UserLibrary (Arq)

UserLibrary_Arq.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,14 +31,16 @@
  */
 
 
+
+
 #import "UserLibrary_Arq.h"
 
 
 @implementation UserLibrary (Arq)
 + (NSString *)arqUserLibraryPath {
-    return [[NSHomeDirectory() stringByAppendingPathComponent:@"Library"] stringByAppendingPathComponent:@"Arq"];
+    return [[NSHomeDirectory() stringByAppendingPathComponent:@"Library"] stringByAppendingPathComponent:@"arq_restore"];
 }
 + (NSString *)arqCachePath {
-    return [NSString stringWithFormat:@"%@/Cache.noindex", [UserLibrary arqUserLibraryPath]];
+    return [NSString stringWithFormat:@"%@/Library/Caches/arq_restore", NSHomeDirectory()];
 }
 @end

arq_restore.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,6 +31,7 @@
  */
 
 
+
 #include <libgen.h>
 #import <Foundation/Foundation.h>
 #import "ArqRestoreCommand.h"
@@ -38,17 +39,19 @@
 
 static void printUsage(const char *exeName) {
 	fprintf(stderr, "Usage:\n");
-    fprintf(stderr, "\t%s [-l log_level] listcomputers <target_type> <target_params>\n", exeName);
+    fprintf(stderr, "\t%s [-l loglevel] listtargets\n", exeName);
-    fprintf(stderr, "\t%s [-l log_level] listfolders <computer_uuid> <encryption_password> <target_type> <target_params>\n", exeName);
+    fprintf(stderr, "\t%s [-l loglevel] addtarget <nickname> aws <access_key> <secret_key> \n", exeName);
-    fprintf(stderr, "\t%s [-l log_level] restore <computer_uuid> <encryption_password> <folder_uuid> <bytes_per_second> <target_type> <target_params>\n", exeName);
+    fprintf(stderr, "\t%s [-l loglevel] addtarget <nickname> local <path>\n", exeName);
-    fprintf(stderr, "\t\ntarget_params by target type:\n");
+    fprintf(stderr, "\t%s [-l loglevel] deletetarget <nickname>\n", exeName);
-    fprintf(stderr, "\taws:                 access_key secret_key bucket_name\n");
+    fprintf(stderr, "\n");
-    fprintf(stderr, "\tsftp:                hostname port path username password_or_keyfile [keyfile_passphrase]\n");
+    fprintf(stderr, "\t%s [-l loglevel] listcomputers <target_nickname>\n", exeName);
-    fprintf(stderr, "\tgreenqloud:          access_key secret_key bucket_name\n");
+    fprintf(stderr, "\t%s [-l loglevel] listfolders <target_nickname> <computer_uuid> <encryption_password>\n", exeName);
-    fprintf(stderr, "\tdreamobjects:        public_key secret_key bucket_name\n");
+    fprintf(stderr, "\t%s [-l loglevel] listtree <target_nickname> <computer_uuid> <encryption_password> <folder_uuid>\n", exeName);
-    fprintf(stderr, "\tgooglecloudstorage:  public_key secret_key bucket_name\n");
+    fprintf(stderr, "\t%s [-l loglevel] restore <target_nickname> <computer_uuid> <encryption_password> <folder_uuid>\n", exeName);
-    fprintf(stderr, "\ts3compatible:        service_url access_key secret_key bucket_name\n");
+    fprintf(stderr, "\t%s [-l loglevel] clearcache <target_nickname>\n", exeName);
-    fprintf(stderr, "\tgoogledrive:         refresh_token path\n");
+    fprintf(stderr, "\n");
+    fprintf(stderr, "log levels: none, error, warn, info, and debug\n");
+    fprintf(stderr, "log output: ~/Library/Logs/arq_restorer\n");
 }
 int main (int argc, const char **argv) {
     char *exePath = strdup(argv[0]);

arq_restore_Prefix.pch

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #ifdef __OBJC__
 #import <Foundation/Foundation.h>
 #import "HSLog.h"

cocoastack/Item.h

@@ -0,0 +1,61 @@
+/*
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
+ 
+ All rights reserved.
+ 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ 
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 
+ * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
+ their contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+@class BufferedInputStream;
+
+
+@interface Item : NSObject <NSCopying> {
+    NSString *itemId;
+    NSString *parentId;
+    NSString *name;
+    BOOL isDirectory;
+    unsigned long long fileSize;
+    NSDate *fileLastModified;
+    NSDate *createdDate; // Not always available. Used in Google Drive.
+    NSString *storageClass;
+    NSString *checksum;
+}
+
+
+@property (retain) NSString *itemId;
+@property (retain) NSString *parentId;
+@property (retain) NSString *name;
+@property BOOL isDirectory;
+@property unsigned long long fileSize;
+@property (retain) NSDate *fileLastModified;
+@property (retain) NSDate *createdDate;
+@property (retain) NSString *storageClass;
+@property (retain) NSString *checksum;
+
+@end

cocoastack/Item.m

@@ -0,0 +1,88 @@
+/*
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
+ 
+ All rights reserved.
+ 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ 
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 
+ * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
+ their contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+
+#import "Item.h"
+#import "StringIO.h"
+#import "IntegerIO.h"
+#import "BooleanIO.h"
+#import "DateIO.h"
+
+
+@implementation Item
+@synthesize itemId;
+@synthesize parentId;
+@synthesize name;
+@synthesize isDirectory;
+@synthesize fileSize;
+@synthesize fileLastModified;
+@synthesize storageClass;
+@synthesize checksum;
+@synthesize createdDate;
+
+
+- (void)dealloc {
+    [itemId release];
+    [parentId release];
+    [name release];
+    [fileLastModified release];
+    [createdDate release];
+    [storageClass release];
+    [checksum release];
+    [super dealloc];
+}
+
+
+#pragma mark NSCopying
+- (id)copyWithZone:(NSZone *)zone {
+    Item *ret = [[Item alloc] init];
+    ret.itemId = itemId;
+    ret.parentId = parentId;
+    ret.name = name;
+    ret.isDirectory = isDirectory;
+    ret.fileSize = fileSize;
+    ret.fileLastModified = fileLastModified;
+    ret.createdDate = createdDate;
+    ret.storageClass = storageClass;
+    ret.checksum = checksum;
+    
+    return ret;
+}
+
+
+#pragma mark NSObject
+- (NSString *)description {
+    return [NSString stringWithFormat:@"<Item: name=%@,id=%@,parent=%@,isDir=%@,fileSize=%qu>", name, itemId, parentId, (isDirectory ? @"YES":@"NO"), fileSize];
+}
+@end

cocoastack/ItemsDB.h

@@ -0,0 +1,62 @@
+/*
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
+ 
+ All rights reserved.
+ 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ 
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 
+ * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
+ their contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+#import "CWLSynthesizeSingleton.h"
+@class Item;
+@class FMDatabase;
+
+
+@interface ItemsDB : NSObject {
+    NSLock *lock;
+    NSMutableDictionary *targetItemsDBsByUUID;
+}
+CWL_DECLARE_SINGLETON_FOR_CLASS(ItemsDB)
+
++ (NSString *)errorDomain;
+
+- (Item *)itemAtPath:(NSString *)thePath targetUUID:(NSString *)theTargetUUID error:(NSError **)error;
+- (NSNumber *)cacheIsLoadedForDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error;
+- (NSMutableDictionary *)itemsByNameInDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error;
+- (BOOL)setItemsByName:(NSDictionary *)theItemsByName inDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error;
+- (BOOL)clearItemsByNameInDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error;
+- (BOOL)destroyForTargetUUID:(NSString *)theTargetUUID error:(NSError **)error;
+- (BOOL)addItem:(Item *)theItem inDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error;
+- (BOOL)addOrReplaceItem:(Item *)theItem inDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error;
+- (BOOL)removeItemWithName:(NSString *)theItemName inDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error;
+- (BOOL)moveItem:(Item *)theItem fromDirectory:(NSString *)theFromDirectory toDirectory:(NSString *)theToDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error;
+- (BOOL)clearReferenceCountsForTargetUUID:(NSString *)theTargetUUID error:(NSError **)error;
+- (BOOL)setReferenceCountOfFileAtPath:(NSString *)thePath targetUUID:(NSString *)theTargetUUID error:(NSError **)error;
+- (NSNumber *)totalSizeOfReferencedFilesInDirectory:(NSString *)theDir targetUUID:(NSString *)theTargetUUID error:(NSError **)error;
+- (NSArray *)pathsOfUnreferencedFilesInDirectory:(NSString *)theDir targetUUID:(NSString *)theTargetUUID error:(NSError **)error;
+@end

cocoastack/ItemsDB.m

@@ -0,0 +1,160 @@
+/*
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
+ 
+ All rights reserved.
+ 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ 
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 
+ * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
+ their contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+#import "ItemsDB.h"
+#import "TargetItemsDB.h"
+
+
+@implementation ItemsDB
+CWL_SYNTHESIZE_SINGLETON_FOR_CLASS(ItemsDB)
+
++ (NSString *)errorDomain {
+    return @"ItemsDBErrorDomain";
+}
+
+
+- (id)init {
+    if (self = [super init]) {
+        lock = [[NSLock alloc] init];
+        [lock setName:@"ItemsDB lock"];
+        targetItemsDBsByUUID = [[NSMutableDictionary alloc] init];
+    }
+    return self;
+}
+
+- (Item *)itemAtPath:(NSString *)thePath targetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    [lock lock];
+    Item *ret = [[self targetItemsDBForTargetUUID:theTargetUUID error:error] itemAtPath:thePath error:error];
+    [lock unlock];
+    return ret;
+}
+- (NSNumber *)cacheIsLoadedForDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    NSNumber *ret = nil;
+    [lock lock];
+    TargetItemsDB *tidb = [self targetItemsDBForTargetUUID:theTargetUUID error:error];
+    if (tidb != nil) {
+        ret = [tidb cacheIsLoadedForDirectory:theDirectory error:error];
+    }
+    [lock unlock];
+    return ret;
+}
+- (NSMutableDictionary *)itemsByNameInDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    [lock lock];
+    NSMutableDictionary *ret = [[self targetItemsDBForTargetUUID:theTargetUUID error:error] itemsByNameInDirectory:theDirectory error:error];
+    [lock unlock];
+    return ret;
+}
+- (BOOL)setItemsByName:(NSDictionary *)theItemsByName inDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    [lock lock];
+    BOOL ret = [[self targetItemsDBForTargetUUID:theTargetUUID error:error] setItemsByName:theItemsByName inDirectory:theDirectory error:error];
+    [lock unlock];
+    return ret;
+}
+- (BOOL)clearItemsByNameInDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    [lock lock];
+    BOOL ret = [[self targetItemsDBForTargetUUID:theTargetUUID error:error] clearItemsByNameInDirectory:theDirectory error:error];
+    [lock unlock];
+    return ret;
+}
+
+
+- (BOOL)destroyForTargetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    [lock lock];
+    BOOL ret = [[self targetItemsDBForTargetUUID:theTargetUUID error:error] destroy:error];
+    [targetItemsDBsByUUID removeObjectForKey:theTargetUUID];
+    [lock unlock];
+    return ret;
+}
+
+- (BOOL)addItem:(Item *)theItem inDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    [lock lock];
+    BOOL ret = [[self targetItemsDBForTargetUUID:theTargetUUID error:error] addItem:theItem inDirectory:theDirectory error:error];
+    [lock unlock];
+    return ret;
+}
+- (BOOL)addOrReplaceItem:(Item *)theItem inDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    [lock lock];
+    BOOL ret = [[self targetItemsDBForTargetUUID:theTargetUUID error:error] addOrReplaceItem:theItem inDirectory:theDirectory error:error];
+    [lock unlock];
+    return ret;
+}
+- (BOOL)removeItemWithName:(NSString *)theItemName inDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    [lock lock];
+    BOOL ret = [[self targetItemsDBForTargetUUID:theTargetUUID error:error] removeItemWithName:theItemName inDirectory:theDirectory error:error];
+    [lock unlock];
+    return ret;
+}
+- (BOOL)moveItem:(Item *)theItem fromDirectory:(NSString *)theFromDirectory toDirectory:(NSString *)theToDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    [lock lock];
+    BOOL ret = [[self targetItemsDBForTargetUUID:theTargetUUID error:error] moveItem:theItem fromDirectory:theFromDirectory toDirectory:theToDirectory error:error];
+    [lock unlock];
+    return ret;
+}
+
+- (BOOL)clearReferenceCountsForTargetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    [lock lock];
+    BOOL ret = [[self targetItemsDBForTargetUUID:theTargetUUID error:error] clearReferenceCounts:error];
+    [lock unlock];
+    return ret;
+}
+- (BOOL)setReferenceCountOfFileAtPath:(NSString *)thePath targetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    [lock lock];
+    BOOL ret = [[self targetItemsDBForTargetUUID:theTargetUUID error:error] setReferenceCountOfFileAtPath:thePath error:error];
+    [lock unlock];
+    return ret;
+}
+- (NSNumber *)totalSizeOfReferencedFilesInDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    [lock lock];
+    NSNumber *ret = [[self targetItemsDBForTargetUUID:theTargetUUID error:error] totalSizeOfReferencedFilesInDirectory:theDirectory error:error];
+    [lock unlock];
+    return ret;
+}
+- (NSArray *)pathsOfUnreferencedFilesInDirectory:(NSString *)theDirectory targetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    [lock lock];
+    NSArray *ret = [[self targetItemsDBForTargetUUID:theTargetUUID error:error] pathsOfUnreferencedFilesInDirectory:theDirectory error:error];
+    [lock unlock];
+    return ret;
+}
+- (TargetItemsDB *)targetItemsDBForTargetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    TargetItemsDB *ret = [targetItemsDBsByUUID objectForKey:theTargetUUID];
+    if (ret == nil) {
+        ret = [[[TargetItemsDB alloc] initWithTargetUUID:theTargetUUID error:error] autorelease];
+        if (ret == nil) {
+            return nil;
+        }
+        [targetItemsDBsByUUID setObject:ret forKey:theTargetUUID];
+    }
+    return ret;
+}
+@end

cocoastack/TargetItemsDB.h

@@ -0,0 +1,60 @@
+/*
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
+ 
+ All rights reserved.
+ 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ 
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 
+ * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
+ their contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+@class Item;
+@class FMDatabaseQueue;
+
+
+@interface TargetItemsDB : NSObject {
+    NSString *dbPath;
+    NSString *lockFilePath;
+    FMDatabaseQueue *fmdbq;
+}
+- (id)initWithTargetUUID:(NSString *)theTargetUUID error:(NSError **)error;
+
+- (Item *)itemAtPath:(NSString *)thePath error:(NSError **)error;
+- (NSNumber *)cacheIsLoadedForDirectory:(NSString *)theDirectory error:(NSError **)error;
+- (NSMutableDictionary *)itemsByNameInDirectory:(NSString *)theDirectory error:(NSError **)error;
+- (BOOL)setItemsByName:(NSDictionary *)theItemsByName inDirectory:(NSString *)theDirectory error:(NSError **)error;
+- (BOOL)clearItemsByNameInDirectory:(NSString *)theDirectory error:(NSError **)error;
+- (BOOL)destroy:(NSError **)error;
+- (BOOL)addItem:(Item *)theItem inDirectory:(NSString *)theDirectory error:(NSError **)error;
+- (BOOL)addOrReplaceItem:(Item *)theItem inDirectory:(NSString *)theDirectory error:(NSError **)error;
+- (BOOL)removeItemWithName:(NSString *)theItemName inDirectory:(NSString *)theDirectory error:(NSError **)error;
+- (BOOL)moveItem:(Item *)theItem fromDirectory:(NSString *)theFromDirectory toDirectory:(NSString *)theToDirectory error:(NSError **)error;
+- (BOOL)clearReferenceCounts:(NSError **)error;
+- (BOOL)setReferenceCountOfFileAtPath:(NSString *)thePath error:(NSError **)error;
+- (NSNumber *)totalSizeOfReferencedFilesInDirectory:(NSString *)theDir error:(NSError **)error;
+- (NSArray *)pathsOfUnreferencedFilesInDirectory:(NSString *)theDir error:(NSError **)error;
+@end

cocoastack/TargetItemsDB.m

@@ -0,0 +1,848 @@
+/*
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
+ 
+ All rights reserved.
+ 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ 
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 
+ * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
+ their contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#import "sqlite3.h"
+#import "TargetItemsDB.h"
+#import "NSFileManager_extra.h"
+#import "ItemsDB.h"
+#import "FMDB.h"
+#import "UserLibrary_Arq.h"
+#import "Item.h"
+#import "NSString_extra.h"
+#import "CacheOwnership.h"
+#import "FlockFile.h"
+#import "FMDatabaseAdditions.h"
+#import "NSString_slashed.h"
+
+
+@implementation TargetItemsDB
+- (id)initWithTargetUUID:(NSString *)theTargetUUID error:(NSError **)error {
+    if (self = [super init]) {
+        dbPath = [[[[UserLibrary arqCachePath] stringByAppendingPathComponent:theTargetUUID] stringByAppendingPathComponent:@"items.db"] retain];
+        lockFilePath = [[dbPath stringByAppendingString:@".lock"] retain];
+        if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:dbPath targetUID:[[CacheOwnership sharedCacheOwnership] uid] targetGID:[[CacheOwnership sharedCacheOwnership] gid] error:error]) {
+            [self release];
+            return nil;
+        }
+        
+        NSError *myError = nil;
+        fmdbq = [[self initDB:&myError] retain];
+        if (fmdbq == nil) {
+            HSLogError(@"failed to open items cache database %@: %@", dbPath, myError);
+            if ([myError isErrorWithDomain:[ItemsDB errorDomain] code:SQLITE_CORRUPT]) {
+                // Delete the file.
+                HSLogInfo(@"deleting corrupt items cache database %@", dbPath);
+                if (![[NSFileManager defaultManager] removeItemAtPath:dbPath error:&myError]) {
+                    HSLogError(@"failed to delete corrupt sqlite database %@: %@", dbPath, myError);
+                }
+                fmdbq = [[self initDB:&myError] retain];
+            }
+        }
+        if (fmdbq == nil) {
+            SETERRORFROMMYERROR;
+            [self release];
+            return nil;
+        }
+    }
+    return self;
+}
+- (void)dealloc {
+    [fmdbq close];
+    [fmdbq release];
+    [dbPath release];
+    [lockFilePath release];
+    [super dealloc];
+}
+
+- (Item *)itemAtPath:(NSString *)thePath error:(NSError **)error {
+    FlockFile *ff = [[[FlockFile alloc] initWithPath:lockFilePath] autorelease];
+    __block Item *ret = nil;
+    if (![ff lockAndExecute:^void() { ret = [self lockedItemAtPath:thePath error:error]; } error:error]) {
+        ret = nil;
+    }
+    return ret;
+}
+- (NSNumber *)cacheIsLoadedForDirectory:(NSString *)theDirectory error:(NSError **)error {
+    // Don't get a flockfile first for performance reasons.
+    
+    __block NSNumber *ret = nil;
+
+    [fmdbq inDatabase:^(FMDatabase *db) {
+        ret = [self isLoadedForDirectory:theDirectory db:db error:error];
+    }];
+
+    return ret;
+}
+- (NSMutableDictionary *)itemsByNameInDirectory:(NSString *)theDirectory error:(NSError **)error {
+    FlockFile *ff = [[[FlockFile alloc] initWithPath:lockFilePath] autorelease];
+    __block NSMutableDictionary *ret = nil;
+    if (![ff lockAndExecute:^void() { ret = [self lockedItemsByNameInDirectory:theDirectory error:error]; } error:error]) {
+        ret = nil;
+    }
+    return ret;
+}
+- (BOOL)setItemsByName:(NSDictionary *)theItemsByName inDirectory:(NSString *)theDirectory error:(NSError **)error {
+    FlockFile *ff = [[[FlockFile alloc] initWithPath:lockFilePath] autorelease];
+    __block BOOL ret = NO;
+    if (![ff lockAndExecute:^void() { ret = [self lockedSetItemsByName:theItemsByName inDirectory:theDirectory error:error]; } error:error]) {
+        ret = NO;
+    }
+    return ret;
+}
+
+- (BOOL)clearItemsByNameInDirectory:(NSString *)theDirectory error:(NSError **)error {
+    FlockFile *ff = [[[FlockFile alloc] initWithPath:lockFilePath] autorelease];
+    __block BOOL ret = NO;
+    if (![ff lockAndExecute:^void() { ret = [self lockedClearItemsByNameInDirectory:theDirectory error:error]; } error:error]) {
+        ret = NO;
+    }
+    return ret;
+}
+- (BOOL)destroy:(NSError **)error {
+    FlockFile *ff = [[[FlockFile alloc] initWithPath:lockFilePath] autorelease];
+    __block BOOL ret = NO;
+    if (![ff lockAndExecute:^void() { ret = [self lockedDestroy:error]; } error:error]) {
+        ret = NO;
+    }
+    return ret;
+}
+
+- (BOOL)addItem:(Item *)theItem inDirectory:(NSString *)theDirectory error:(NSError **)error {
+    FlockFile *ff = [[[FlockFile alloc] initWithPath:lockFilePath] autorelease];
+    __block BOOL ret = NO;
+    if (![ff lockAndExecute:^void() { ret = [self lockedAddItem:theItem inDirectory:theDirectory error:error]; } error:error]) {
+        ret = NO;
+    }
+    return ret;
+}
+- (BOOL)addOrReplaceItem:(Item *)theItem inDirectory:(NSString *)theDirectory error:(NSError **)error {
+    FlockFile *ff = [[[FlockFile alloc] initWithPath:lockFilePath] autorelease];
+    __block BOOL ret = NO;
+    if (![ff lockAndExecute:^void() { ret = [self lockedAddOrReplaceItem:theItem inDirectory:theDirectory error:error]; } error:error]) {
+        ret = NO;
+    }
+    return ret;
+}
+- (BOOL)removeItemWithName:(NSString *)theItemName inDirectory:(NSString *)theDirectory error:(NSError **)error {
+    FlockFile *ff = [[[FlockFile alloc] initWithPath:lockFilePath] autorelease];
+    __block BOOL ret = NO;
+    if (![ff lockAndExecute:^void() { ret = [self lockedRemoveItemWithName:theItemName inDirectory:theDirectory error:error]; } error:error]) {
+        ret = NO;
+    }
+    return ret;
+}
+- (BOOL)moveItem:(Item *)theItem fromDirectory:(NSString *)theFromDirectory toDirectory:(NSString *)theToDirectory error:(NSError **)error {
+    FlockFile *ff = [[[FlockFile alloc] initWithPath:lockFilePath] autorelease];
+    __block BOOL ret = NO;
+    if (![ff lockAndExecute:^void() { ret = [self lockedMoveItem:theItem fromDirectory:theFromDirectory toDirectory:theToDirectory error:error]; } error:error]) {
+        ret = NO;
+    }
+    return ret;
+}
+- (BOOL)clearReferenceCounts:(NSError **)error {
+    FlockFile *ff = [[[FlockFile alloc] initWithPath:lockFilePath] autorelease];
+    __block BOOL ret = NO;
+    if (![ff lockAndExecute:^void() { ret = [self lockedClearReferenceCounts:error]; } error:error]) {
+        ret = NO;
+    }
+    return ret;
+}
+- (BOOL)setReferenceCountOfFileAtPath:(NSString *)thePath error:(NSError **)error {
+    FlockFile *ff = [[[FlockFile alloc] initWithPath:lockFilePath] autorelease];
+    __block BOOL ret = NO;
+    if (![ff lockAndExecute:^void() { ret = [self lockedSetReferenceCountOfFileAtPath:thePath error:error]; } error:error]) {
+        ret = NO;
+    }
+    return ret;
+}
+- (NSNumber *)totalSizeOfReferencedFilesInDirectory:(NSString *)theDir error:(NSError **)error {
+    FlockFile *ff = [[[FlockFile alloc] initWithPath:lockFilePath] autorelease];
+    __block NSNumber *ret = nil;
+    if (![ff lockAndExecute:^void() { ret = [self lockedTotalSizeOfReferencedFilesInDirectory:theDir error:error]; } error:error]) {
+        ret = nil;
+    }
+    return ret;
+}
+- (NSArray *)pathsOfUnreferencedFilesInDirectory:(NSString *)theDir error:(NSError **)error {
+    FlockFile *ff = [[[FlockFile alloc] initWithPath:lockFilePath] autorelease];
+    __block NSArray *ret = nil;
+    if (![ff lockAndExecute:^void() { ret = [self lockedPathsOfUnreferencedFilesInDirectory:theDir error:error]; } error:error]) {
+        ret = nil;
+    }
+    return ret;
+}
+
+
+#pragma mark internal
+- (BOOL)lockedSetReferenceCountOfFileAtPath:(NSString *)thePath error:(NSError **)error {
+    __block BOOL ret = NO;
+    
+    [fmdbq inDatabase:^(FMDatabase *db) {
+        if (![db executeUpdate:@"UPDATE items SET refcount = 1 WHERE path = ?" withArgumentsInArray:[NSArray arrayWithObject:thePath]]) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db update refcount: error=%@, db=%@", [db lastErrorMessage], dbPath);
+            return;
+        }
+//        if ([db changes] != 0) {
+//            HSLogDebug(@"set refcount for %@", thePath);
+//        }
+        ret = YES;
+    }];
+    return ret;
+}
+- (NSNumber *)lockedTotalSizeOfReferencedFilesInDirectory:(NSString *)theDir error:(NSError **)error {
+    __block NSNumber *ret = nil;
+    
+    theDir = [theDir slashed];
+    [fmdbq inDatabase:^(FMDatabase *db) {
+        NSString *theLikeParam = [theDir stringByAppendingString:@"%"];
+        FMResultSet *rs = [db executeQuery:@"SELECT SUM(file_size) FROM items WHERE refcount > 0 AND slashed_directory LIKE ?" withArgumentsInArray:[NSArray arrayWithObject:theLikeParam]];
+        if (rs == nil) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db select sum(file_size): error=%@, db=%@", [db lastErrorMessage], dbPath);
+            return;
+        }
+        [rs next];
+        unsigned long long total = [rs unsignedLongLongIntForColumnIndex:0];
+        ret = [NSNumber numberWithUnsignedLongLong:total];
+        [rs close];
+    }];
+    
+    return ret;
+}
+- (NSArray *)lockedPathsOfUnreferencedFilesInDirectory:(NSString *)theDir error:(NSError **)error {
+    __block NSMutableArray *ret = nil;
+    
+    theDir = [theDir slashed];
+    [fmdbq inDatabase:^(FMDatabase *db) {
+        NSString *theLikeParam = [theDir stringByAppendingString:@"%"];
+        FMResultSet *rs = [db executeQuery:@"SELECT path FROM items WHERE refcount = 0 AND is_directory = 0 AND slashed_directory LIKE ?" withArgumentsInArray:[NSArray arrayWithObject:theLikeParam]];
+        if (rs == nil) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db select unreferenced paths: error=%@, db=%@", [db lastErrorMessage], dbPath);
+            return;
+        }
+        ret = [NSMutableArray array];
+        while ([rs next]) {
+            [ret addObject:[rs stringForColumnIndex:0]];
+        }
+        [rs close];
+    }];
+    
+    return ret;
+}
+- (Item *)lockedItemAtPath:(NSString *)thePath error:(NSError **)error {
+    NSString *theDirectory = [thePath stringByDeletingLastPathComponent];
+    
+    __block Item *ret = nil;
+    
+    [fmdbq inDatabase:^(FMDatabase *db) {
+        NSNumber *loaded = [self isLoadedForDirectory:theDirectory db:db error:error];
+        if (loaded == nil) {
+            return;
+        }
+        if (![loaded boolValue]) {
+            SETNSERROR([ItemsDB errorDomain], ERROR_CACHE_NOT_LOADED, @"itemAtPath: cache not loaded for directory %@", theDirectory);
+            return;
+        }
+        
+//        HSLogDebug(@"querying for item at path %@", thePath);
+//        NSTimeInterval theTime = [NSDate timeIntervalSinceReferenceDate];
+        FMResultSet *rs = [db executeQuery:@"SELECT name, item_id, parent_id, is_directory, file_size, file_last_modified, storage_class, checksum FROM items WHERE path = ?" withArgumentsInArray:[NSArray arrayWithObject:thePath]];
+//        HSLogDebug(@"query for item at path %@ took %0.2f seconds", thePath, ([NSDate timeIntervalSinceReferenceDate] - theTime));
+        if (rs == nil) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db select item at path: error=%@, db=%@", [db lastErrorMessage], dbPath);
+            return;
+        }
+        if (![rs next]) {
+            SETNSERROR([ItemsDB errorDomain], ERROR_NOT_FOUND, @"%@ not found", thePath);
+            HSLogDebug(@"%@ not found in cached list of items", thePath);
+        } else {
+            ret = [[[Item alloc] init] autorelease];
+            ret.name = [rs stringForColumnIndex:0];
+            ret.itemId = [rs stringForColumnIndex:1];
+            ret.parentId = [rs stringForColumnIndex:2];
+            ret.isDirectory = [rs boolForColumnIndex:3];
+            ret.fileSize = [rs intForColumnIndex:4];
+            ret.fileLastModified = [rs dateForColumnIndex:5];
+            ret.storageClass = [rs stringForColumnIndex:6];
+            ret.checksum = [rs stringForColumnIndex:7];
+        }
+        [rs close];
+    }];
+    
+    return ret;
+}
+- (NSMutableDictionary *)lockedItemsByNameInDirectory:(NSString *)theDirectory error:(NSError **)error {
+    __block NSMutableDictionary *ret = nil;
+    
+    [fmdbq inDatabase:^(FMDatabase *db) {
+        NSNumber *loaded = [self isLoadedForDirectory:theDirectory db:db error:error];
+        if (loaded == nil) {
+            return;
+        }
+        if (![loaded boolValue]) {
+            SETNSERROR([ItemsDB errorDomain], ERROR_CACHE_NOT_LOADED, @"itemsByNameInDirectory: cache not loaded for directory %@", theDirectory);
+            return;
+        }
+        
+        NSString *slashedDirectory = [theDirectory stringByAppendingTrailingSlash];
+        
+//        HSLogDebug(@"query for items in directory %@", theDirectory);
+//        NSTimeInterval theTime = [NSDate timeIntervalSinceReferenceDate];
+        FMResultSet *rs = [db executeQuery:@"SELECT name, item_id, parent_id, is_directory, file_size, file_last_modified, storage_class, checksum FROM items WHERE slashed_directory = ?" withArgumentsInArray:[NSArray arrayWithObject:slashedDirectory]];
+//        HSLogDebug(@"query for items in directory %@ took %0.2f seconds", theDirectory, ([NSDate timeIntervalSinceReferenceDate] - theTime));
+        if (rs == nil) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db select items: error=%@, db=%@", [db lastErrorMessage], dbPath);
+            return;
+        }
+        ret = [NSMutableDictionary dictionary];
+        while ([rs next]) {
+            Item *item = [[[Item alloc] init] autorelease];
+            item.name = [rs stringForColumnIndex:0];
+            item.itemId = [rs stringForColumnIndex:1];
+            item.parentId = [rs stringForColumnIndex:2];
+            item.isDirectory = [rs boolForColumnIndex:3];
+            item.fileSize = [rs intForColumnIndex:4];
+            item.fileLastModified = [rs dateForColumnIndex:5];
+            item.storageClass = [rs stringForColumnIndex:6];
+            item.checksum = [rs stringForColumnIndex:7];
+            
+            [ret setObject:item forKey:item.name];
+        }
+        [rs close];
+    }];
+    
+    return ret;
+}
+- (BOOL)lockedSetItemsByName:(NSDictionary *)theItemsByName inDirectory:(NSString *)theDirectory error:(NSError **)error {
+    __block BOOL ret = NO;
+    
+    [fmdbq inDatabase:^(FMDatabase *db) {
+        if (![db beginTransaction]) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"begin transaction in setItemsByName failed: error=%@, db=%@", [db lastErrorMessage], dbPath);
+            return;
+        }
+        ret = [self doSetItemsByName:theItemsByName inDirectory:theDirectory database:db error:error];
+        if (ret) {
+            if (![db commit]) {
+                SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"commit in setItemsByName failed: error=%@, db=%@", [db lastErrorMessage], dbPath);
+                ret = NO;
+                return;
+            }
+        } else {
+            [db rollback];
+        }
+    }];
+    
+    return ret;
+}
+- (BOOL)doSetItemsByName:(NSDictionary *)theItemsByName inDirectory:(NSString *)theDirectory database:(FMDatabase *)db error:(NSError **)error {
+    NSString *slashedDirectory = [theDirectory stringByAppendingTrailingSlash];
+    
+    NSNumber *loaded = [self isLoadedForDirectory:theDirectory db:db error:error];
+    if (loaded == nil) {
+        return NO;
+    }
+    if (![loaded boolValue]) {
+        if (![db executeUpdate:@"INSERT INTO loaded_directories (path) VALUES (?)" withArgumentsInArray:[NSArray arrayWithObject:theDirectory]]) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"insert into loaded_directories: error=%@, db=%@", [db lastErrorMessage], dbPath);
+            return NO;
+        }
+    }
+    if (![db executeUpdate:@"DELETE FROM items WHERE slashed_directory = ?" withArgumentsInArray:[NSArray arrayWithObject:slashedDirectory]]) {
+        SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"delete items: error=%@, db=%@", [db lastErrorMessage], dbPath);
+        return NO;
+    }
+    HSLogDebug(@"inserting %ld rows into the cache db for %@", [theItemsByName count], theDirectory);
+    for (Item *item in [theItemsByName allValues]) {
+        if (![self deleteAndInsertItem:item inDirectory:slashedDirectory db:db error:error]) {
+            return NO;
+        }
+    }
+    HSLogDebug(@"inserted %ld rows into the cache db for %@", [theItemsByName count], theDirectory);
+    
+    return YES;
+}
+- (BOOL)lockedClearItemsByNameInDirectory:(NSString *)theDirectory error:(NSError **)error {
+    HSLogDebug(@"deleting cached data for %@ and all subdirectories", theDirectory);
+    
+    NSString *slashedDirectory = [theDirectory stringByAppendingTrailingSlash];
+    
+    __block BOOL ret = NO;
+    
+    [fmdbq inDatabase:^(FMDatabase *db) {
+        // Delete loaded_directories.
+        NSArray *args = [NSArray arrayWithObjects:theDirectory, [slashedDirectory stringByAppendingString:@"%"], nil];
+        if (![db executeUpdate:@"DELETE FROM loaded_directories WHERE path = ? OR path LIKE ?" withArgumentsInArray:args]) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"delete from loaded_directories for %@: error=%@, db=%@", theDirectory, [db lastErrorMessage], dbPath);
+            return;
+        }
+        
+        // Delete items.
+        if (![db executeUpdate:@"DELETE FROM items WHERE slashed_directory LIKE ?" withArgumentsInArray:[NSArray arrayWithObject:[slashedDirectory stringByAppendingString:@"%"]]]) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"delete from items for %@: error=%@, db=%@", theDirectory, [db lastErrorMessage], dbPath);
+            return;
+        }
+        
+        ret = YES;
+    }];
+    
+    return ret;
+}
+- (BOOL)lockedDestroy:(NSError **)error {
+    if ([[NSFileManager defaultManager] fileExistsAtPath:dbPath]) {
+        if (![[NSFileManager defaultManager] removeItemAtPath:dbPath error:error]) {
+            return NO;
+        }
+    }
+    return YES;
+}
+- (BOOL)lockedAddItem:(Item *)theItem inDirectory:(NSString *)theDirectory error:(NSError **)error {
+    __block BOOL ret = NO;
+    
+    [fmdbq inDatabase:^(FMDatabase *db) {
+        NSNumber *loaded = [self isLoadedForDirectory:theDirectory db:db error:error];
+        if (loaded == nil) {
+            return;
+        }
+        if (![loaded boolValue]) {
+            SETNSERROR([ItemsDB errorDomain], ERROR_CACHE_NOT_LOADED, @"addItem: cache not loaded for directory %@", theDirectory);
+            return;
+        }
+        
+        if (![db beginTransaction]) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"begin transaction in addItem failed: error=%@, db=%@", [db lastErrorMessage], dbPath);
+            return;
+        }
+        ret = [self insertItem:theItem inDirectory:theDirectory db:db error:error];
+        if (ret) {
+            if (![db commit]) {
+                SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"commit in addItem failed: error=%@, db=%@", [db lastErrorMessage], dbPath);
+                ret = NO;
+                return;
+            }
+        } else {
+            [db rollback];
+        }
+    }];
+    
+    return ret;
+}
+- (BOOL)lockedAddOrReplaceItem:(Item *)theItem inDirectory:(NSString *)theDirectory error:(NSError **)error {
+    __block BOOL ret = NO;
+    
+    [fmdbq inDatabase:^(FMDatabase *db) {
+        NSNumber *loaded = [self isLoadedForDirectory:theDirectory db:db error:error];
+        if (loaded == nil) {
+            return;
+        }
+        if (![loaded boolValue]) {
+            SETNSERROR([ItemsDB errorDomain], ERROR_CACHE_NOT_LOADED, @"addOrReplaceItem: cache not loaded for directory %@", theDirectory);
+            return;
+        }
+        
+        if (![db beginTransaction]) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"begin transaction in addOrReplaceItem failed: error=%@, db=%@", [db lastErrorMessage], dbPath);
+            return;
+        }
+        ret = [self deleteAndInsertItem:theItem inDirectory:theDirectory db:db error:error];
+        if (ret) {
+            if (![db commit]) {
+                SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"commit in addOrReplaceItem failed: error=%@, db=%@", [db lastErrorMessage], dbPath);
+                ret = NO;
+                return;
+            }
+        } else {
+            [db rollback];
+        }
+    }];
+    
+    return ret;
+}
+- (BOOL)lockedRemoveItemWithName:(NSString *)theItemName inDirectory:(NSString *)theDirectory error:(NSError **)error {
+    __block BOOL ret = NO;
+    
+    [fmdbq inDatabase:^(FMDatabase *db) {
+        NSNumber *loaded = [self isLoadedForDirectory:theDirectory db:db error:error];
+        if (loaded == nil) {
+            return;
+        }
+        if (![loaded boolValue]) {
+            SETNSERROR([ItemsDB errorDomain], ERROR_CACHE_NOT_LOADED, @"removeItemWithName: cache not loaded for directory %@", theDirectory);
+            return;
+        }
+        
+        NSString *slashedDirectory = [theDirectory stringByAppendingTrailingSlash];
+        if (![db executeUpdate:@"DELETE FROM items WHERE name = ? AND slashed_directory = ?" withArgumentsInArray:[NSArray arrayWithObjects:theItemName, slashedDirectory, nil]]) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"delete from items for %@ in %@: error=%@, db=%@", theItemName, theDirectory, [db lastErrorMessage], dbPath);
+            return;
+        }
+        
+        ret = YES;
+    }];
+    
+    return ret;
+}
+- (BOOL)lockedMoveItem:(Item *)theItem fromDirectory:(NSString *)theFromDirectory toDirectory:(NSString *)theToDirectory error:(NSError **)error {
+    __block BOOL ret = NO;
+    
+    [fmdbq inDatabase:^(FMDatabase *db) {
+        if (![db beginTransaction]) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"begin transaction in setItemsByName failed: error=%@, db=%@", [db lastErrorMessage], dbPath);
+            return;
+        }
+        ret = [self doLockedMoveItem:theItem fromDirectory:theFromDirectory toDirectory:theToDirectory error:error];
+        if (ret) {
+            if (![db commit]) {
+                SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"commit in setItemsByName failed: error=%@, db=%@", [db lastErrorMessage], dbPath);
+                ret = NO;
+                return;
+            }
+        } else {
+            [db rollback];
+        }
+    }];
+    return ret;
+}
+- (BOOL)doLockedMoveItem:(Item *)theItem fromDirectory:(NSString *)theFromDirectory toDirectory:(NSString *)theToDirectory error:(NSError **)error {
+    __block BOOL ret = NO;
+    
+    [fmdbq inDatabase:^(FMDatabase *db) {
+        NSNumber *loaded = [self isLoadedForDirectory:theFromDirectory db:db error:error];
+        if (loaded == nil) {
+            return;
+        }
+        if (![loaded boolValue]) {
+            SETNSERROR([ItemsDB errorDomain], ERROR_CACHE_NOT_LOADED, @"removeItemWithName: cache not loaded for directory %@", theFromDirectory);
+            return;
+        }
+        loaded = [self isLoadedForDirectory:theToDirectory db:db error:error];
+        if (loaded == nil) {
+            return;
+        }
+        if (![loaded boolValue]) {
+            SETNSERROR([ItemsDB errorDomain], ERROR_CACHE_NOT_LOADED, @"removeItemWithName: cache not loaded for directory %@", theToDirectory);
+            return;
+        }
+        
+        NSString *slashedFromDirectory = [theFromDirectory stringByAppendingTrailingSlash];
+        if (![db executeUpdate:@"DELETE FROM items WHERE name = ? AND slashed_directory = ?" withArgumentsInArray:[NSArray arrayWithObjects:theItem.name, slashedFromDirectory, nil]]) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"delete from items for %@ in %@: error=%@, db=%@", theItem.name, theFromDirectory, [db lastErrorMessage], dbPath);
+            return;
+        }
+        
+        if (![self insertItem:theItem inDirectory:theToDirectory db:db error:error]) {
+            return;
+        }
+        ret = YES;
+    }];
+    
+    return ret;
+}
+- (BOOL)lockedClearReferenceCounts:(NSError **)error {
+    __block BOOL ret = NO;
+    
+    [fmdbq inDatabase:^(FMDatabase *db) {
+        if (![db executeUpdate:@"UPDATE items SET refcount = 0"]) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"update refcount to 0 error: %@, db=%@", [db lastErrorMessage], dbPath);
+            return;
+        }
+        int rowsChanged = [db changes];
+        HSLogDebug(@"set refcount to 0 on %d rows", rowsChanged);
+        ret = YES;
+    }];
+    
+    return ret;
+}
+- (BOOL)insertItem:(Item *)theItem inDirectory:(NSString *)theDirectory db:(FMDatabase *)db error:(NSError **)error {
+    NSAssert([db inTransaction], @"must be in a transaction");
+    
+    NSString *thePath = [theDirectory stringByAppendingPathComponent:theItem.name];
+    FMResultSet *rs = [db executeQuery:@"SELECT name, item_id, parent_id, is_directory, file_size, file_last_modified, storage_class, checksum FROM items WHERE path = ?" withArgumentsInArray:[NSArray arrayWithObject:thePath]];
+    if (rs == nil) {
+        SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db select item at path: error=%@, db=%@", [db lastErrorMessage], dbPath);
+        return NO;
+    }
+    Item *existing = nil;
+    if ([rs next]) {
+        existing = [[[Item alloc] init] autorelease];
+        existing.name = [rs stringForColumnIndex:0];
+        existing.itemId = [rs stringForColumnIndex:1];
+        existing.parentId = [rs stringForColumnIndex:2];
+        existing.isDirectory = [rs boolForColumnIndex:3];
+        existing.fileSize = [rs intForColumnIndex:4];
+        existing.fileLastModified = [rs dateForColumnIndex:5];
+        existing.storageClass = [rs stringForColumnIndex:6];
+        existing.checksum = [rs stringForColumnIndex:7];
+    }
+    [rs close];
+    if (existing != nil) {
+        NSDictionary *userInfo = [NSDictionary dictionaryWithObjectsAndKeys:
+                                  @"item exists", NSLocalizedDescriptionKey,
+                                  existing, @"previouslyExistingItem", nil];
+        NSError *myError = [[[NSError alloc] initWithDomain:[ItemsDB errorDomain] code:ERROR_ITEM_EXISTS userInfo:userInfo] autorelease];
+        HSLogDebug(@"insertItem: %@", myError);
+        SETERRORFROMMYERROR;
+        return NO;
+    }
+
+    /* columns:
+     path
+     slashed_directory
+     name
+     item_id
+     parent_id
+     is_directory
+     file_size
+     file_last_modified
+     storage_class
+     checksum
+     */
+    NSArray *args = [NSArray arrayWithObjects:thePath,
+                     [theDirectory stringByAppendingTrailingSlash],
+                     theItem.name,
+                     (theItem.itemId != nil ? theItem.itemId : [NSNull null]),
+                     (theItem.parentId != nil ? theItem.parentId : [NSNull null]),
+                     [NSNumber numberWithInt:(theItem.isDirectory ? 1 : 0)],
+                     [NSNumber numberWithUnsignedLongLong:theItem.fileSize],
+                     (theItem.fileLastModified != nil ? [NSNumber numberWithDouble:[theItem.fileLastModified timeIntervalSince1970]] : [NSNull null]),
+                     (theItem.storageClass != nil ? theItem.storageClass : [NSNull null]),
+                     (theItem.checksum != nil ? theItem.checksum : [NSNull null]),
+                     nil];
+    if (![db executeUpdate:@"INSERT INTO items (path, slashed_directory, name, item_id, parent_id, is_directory, file_size, file_last_modified, storage_class, checksum) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" withArgumentsInArray:args]) {
+        return NO;
+    }
+    return YES;
+}
+- (BOOL)deleteAndInsertItem:(Item *)theItem inDirectory:(NSString *)theDirectory db:(FMDatabase *)db error:(NSError **)error {
+    NSAssert([db inTransaction], @"must be in a transaction");
+    
+    NSString *thePath = [theDirectory stringByAppendingPathComponent:theItem.name];
+    if (![db executeUpdate:@"DELETE FROM items WHERE path = ?" withArgumentsInArray:[NSArray arrayWithObjects:thePath, nil]]) {
+        SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"delete error: error=%@, db=%@", [db lastErrorMessage], dbPath);
+        return NO;
+    }
+    
+    /* columns:
+     path
+     slashed_directory
+     name
+     item_id
+     parent_id
+     is_directory
+     file_size
+     file_last_modified
+     storage_class
+     checksum
+     */
+    NSArray *args = [NSArray arrayWithObjects:thePath,
+                     [theDirectory stringByAppendingTrailingSlash],
+                     theItem.name,
+                     (theItem.itemId != nil ? theItem.itemId : [NSNull null]),
+                     (theItem.parentId != nil ? theItem.parentId : [NSNull null]),
+                     [NSNumber numberWithInt:(theItem.isDirectory ? 1 : 0)],
+                     [NSNumber numberWithUnsignedLongLong:theItem.fileSize],
+                     (theItem.fileLastModified != nil ? [NSNumber numberWithDouble:[theItem.fileLastModified timeIntervalSince1970]] : [NSNull null]),
+                     (theItem.storageClass != nil ? theItem.storageClass : [NSNull null]),
+                     (theItem.checksum != nil ? theItem.checksum : [NSNull null]),
+                     nil];
+    if (![db executeUpdate:@"INSERT INTO items (path, slashed_directory, name, item_id, parent_id, is_directory, file_size, file_last_modified, storage_class, checksum) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" withArgumentsInArray:args]) {
+        return NO;
+    }
+    return YES;
+}
+- (NSNumber *)isLoadedForDirectory:(NSString *)theDirectory db:(FMDatabase *)db error:(NSError **)error {
+    NSNumber *ret = nil;
+    
+//    HSLogDebug(@"query for count of loaded_directories for %@", theDirectory);
+//    NSTimeInterval theTime = [NSDate timeIntervalSinceReferenceDate];
+    FMResultSet *rs = [db executeQuery:@"SELECT COUNT(*) AS COUNT FROM loaded_directories WHERE path = ?" withArgumentsInArray:[NSArray arrayWithObject:theDirectory]];
+//    HSLogDebug(@"query for count of loaded_directories for %@ took %0.2f seconds", theDirectory, ([NSDate timeIntervalSinceReferenceDate] - theTime));
+    if (rs == nil) {
+        SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db select items: error=%@, db=%@", [db lastErrorMessage], dbPath);
+        return nil;
+    }
+    if (![rs next]) {
+        SETNSERROR([ItemsDB errorDomain], -1, @"db select: no row!");
+    } else {
+        int count = [rs intForColumnIndex:0];
+        ret = [NSNumber numberWithBool:(count > 0)];
+    }
+    [rs close];
+    
+    return ret;
+}
+
+- (FMDatabaseQueue *)initDB:(NSError **)error {
+    if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:dbPath targetUID:[[CacheOwnership sharedCacheOwnership] uid] targetGID:[[CacheOwnership sharedCacheOwnership] gid] error:error]) {
+        return nil;
+    }
+    
+    __block BOOL ret = NO;
+    
+    FMDatabaseQueue *dbq = [FMDatabaseQueue databaseQueueWithPath:dbPath];
+    [dbq inDatabase:^(FMDatabase *db) {
+        if (chmod([dbPath fileSystemRepresentation], S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) < 0) {
+            int errnum = errno;
+            HSLogError(@"chmod(%@) error %d: %s", dbPath, errnum, strerror(errnum));
+        }
+        if (chown([dbPath fileSystemRepresentation], [[CacheOwnership sharedCacheOwnership] uid], [[CacheOwnership sharedCacheOwnership] gid]) == -1) {
+            int errnum = errno;
+            SETNSERROR(@"UnixErrorDomain", errnum, @"chown(%@, %d, %d): %s", dbPath, [[CacheOwnership sharedCacheOwnership] uid], [[CacheOwnership sharedCacheOwnership] gid], strerror(errnum));
+            return;
+        }
+        if (![db executeUpdate:@"CREATE TABLE IF NOT EXISTS loaded_directories (path TEXT NOT NULL PRIMARY KEY)"]) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db create table: error=%@, db=%@", [db lastErrorMessage], dbPath);
+            return;
+        }
+        if (![db executeUpdate:@"CREATE TABLE IF NOT EXISTS items (path TEXT NOT NULL PRIMARY KEY, slashed_directory TEXT NOT NULL, name TEXT NOT NULL, item_id TEXT, parent_id TEXT, is_directory INT, file_size INTEGER, file_last_modified REAL, storage_class TEXT, checksum TEXT)"]) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db create table: error=%@, db=%@", [db lastErrorMessage], dbPath);
+            return;
+        }
+        [db setShouldCacheStatements:YES];
+        
+        if (![db columnExists:@"checksum" inTableWithName:@"items"]) {
+            // Update database schema and delete the data.
+            if (![db executeUpdate:@"ALTER TABLE items ADD COLUMN checksum TEXT"]) {
+                SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db alter table: error=%@, db=%@", [db lastErrorMessage], dbPath);
+                return;
+            }
+            if (![db beginTransaction]) {
+                SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db begin transaction: error=%@, db=%@", [db lastErrorMessage], dbPath);
+                return;
+            }
+            BOOL ret = [db executeUpdate:@"DELETE FROM items"]
+            && [db executeUpdate:@"DELETE FROM loaded_directories"];
+            if (ret) {
+                [db commit];
+            } else {
+                SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db delete: error=%@, db=%@", [db lastErrorMessage], dbPath);
+                [db rollback];
+                return;
+            }
+        }
+        
+        if (![db columnExists:@"refcount" inTableWithName:@"items"]) {
+            if (![db executeUpdate:@"ALTER TABLE items ADD COLUMN refcount INTEGER NOT NULL DEFAULT 0"]) {
+                SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db alter table: error=%@, db=%@", [db lastErrorMessage], dbPath);
+                return;
+            }
+        }
+        
+        if ([self schemaVersion] == 0) {
+            if (![db executeUpdate:@"CREATE INDEX items_slashed_directory ON items (slashed_directory)"]) {
+                NSString *msg = [db lastErrorMessage];
+                if ([msg rangeOfString:@"already exists"].location != NSNotFound) {
+                    SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db create index items_slashed_directory: error=%@, db=%@", [db lastErrorMessage], dbPath);
+                    return;
+                } else {
+                    HSLogDebug(@"db create index items_slashed_directory: %@", msg);
+                }
+            }
+            if (![self setSchemaVersion:1 error:error]) {
+                return;
+            }
+        }
+        if ([self schemaVersion] == 1) {
+            if (![db executeUpdate:@"CREATE INDEX items_is_directory ON items (is_directory)"]) {
+                SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db create index items_is_directory: error=%@, db=%@", [db lastErrorMessage], dbPath);
+                return;
+            }
+            if (![self setSchemaVersion:2 error:error]) {
+                return;
+            }
+        }
+        if ([self schemaVersion] == 2) {
+            if (![db executeUpdate:@"CREATE INDEX items_refcount ON items (refcount)"]) {
+                SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db create index items_refcount: error=%@, db=%@", [db lastErrorMessage], dbPath);
+                return;
+            }
+            if (![self setSchemaVersion:3 error:error]) {
+                return;
+            }
+        }
+        ret = YES;
+    }];
+    if (!ret) {
+        return nil;
+    }
+    return dbq;
+}
+
+- (int)schemaVersion {
+    __block int ret = 0;
+    
+    FMDatabaseQueue *dbq = [FMDatabaseQueue databaseQueueWithPath:dbPath];
+    [dbq inDatabase:^(FMDatabase *db) {
+        if (![db tableExists:@"items_schema"]) {
+            if (![db executeUpdate:@"CREATE TABLE IF NOT EXISTS items_schema (version INTEGER NOT NULL PRIMARY KEY)"]) {
+                HSLogError(@"db create table: error=%@, db=%@", [db lastErrorMessage], dbPath);
+                return;
+            }
+        }
+        
+        FMResultSet *rs = [db executeQuery:@"SELECT version FROM items_schema" withArgumentsInArray:[NSArray array]];
+        if (rs == nil) {
+            HSLogError(@"db select version: error=%@, db=%@", [db lastErrorMessage], dbPath);
+            return;
+        }
+        if (![rs next]) {
+            if (![db executeUpdate:@"INSERT INTO items_schema VALUES (0)"]) {
+                HSLogError(@"db insert into items_schema: error=%@, db=%@", [db lastErrorMessage], dbPath);
+                return;
+            }
+        } else {
+            ret = [rs intForColumn:@"version"];
+        }
+        [rs close];
+    }];
+    return ret;
+}
+- (BOOL)setSchemaVersion:(int)theSchemaVersion error:(NSError **)error {
+    int currentVersion = [self schemaVersion]; // This creates the table too, if necessary.
+    HSLogDebug(@"updating items schema version from %d to %d", currentVersion, theSchemaVersion);
+    
+    __block BOOL ret = NO;
+    FMDatabaseQueue *dbq = [FMDatabaseQueue databaseQueueWithPath:dbPath];
+    [dbq inDatabase:^(FMDatabase *db) {
+        if (![db executeUpdate:@"UPDATE items_schema SET version = ?" withArgumentsInArray:[NSArray arrayWithObject:[NSNumber numberWithInt:theSchemaVersion]]]) {
+            SETNSERROR([ItemsDB errorDomain], [db lastErrorCode], @"db update items_schema: error=%@, db=%@", [db lastErrorMessage], dbPath);
+            return;
+        }
+        ret = YES;
+    }];
+    
+    return ret;
+}
+@end

cocoastack/aws/AWSQueryError.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,6 +31,8 @@
  */
 
 
+
+
 @interface AWSQueryError : NSObject <NSXMLParserDelegate> {
     NSMutableDictionary *values;
     NSMutableString *currentStringBuffer;

cocoastack/aws/AWSQueryError.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #import "AWSQueryError.h"
 
 
@@ -43,7 +45,7 @@
         [parser parse];
         [parser release];
         if (parseErrorOccurred) {
-            nsError = [[NSError errorWithDomain:theDomain code:theCode description:@"SNS error"] retain];
+            nsError = [[NSError alloc] initWithDomain:theDomain code:theCode description:@"SNS error"];
         } else {
             NSMutableDictionary *userInfo = [NSMutableDictionary dictionary];
             [userInfo setObject:[NSNumber numberWithInt:theCode] forKey:@"HTTPStatusCode"];
@@ -58,7 +60,7 @@
                 msg = @"Your AWS account is not signed up all services. Please visit http://aws.amazon.com and sign up for S3, Glacier, SNS and SQS.";
             }
             [userInfo setObject:msg forKey:NSLocalizedDescriptionKey];
-            nsError = [[NSError errorWithDomain:theDomain code:theCode userInfo:userInfo] retain];
+            nsError = [[NSError alloc] initWithDomain:theDomain code:theCode userInfo:userInfo];
         }
     }
     return self;

cocoastack/aws/AWSQueryRequest.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 @class AWSQueryResponse;
 
 

cocoastack/aws/AWSQueryRequest.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #import "AWSQueryRequest.h"
 #import "AWSQueryResponse.h"
 #import "HTTPConnectionFactory.h"

cocoastack/aws/AWSQueryResponse.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,6 +31,8 @@
  */
 
 
+
+
 @interface AWSQueryResponse : NSObject {
     int code;
     NSDictionary *headers;

cocoastack/aws/AWSQueryResponse.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #import "AWSQueryResponse.h"
 
 @implementation AWSQueryResponse

cocoastack/aws/AWSRegion.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,6 +31,8 @@
  */
 
 
+
+
 @interface AWSRegion : NSObject {
     NSString *regionName;
     NSArray *s3LocationConstraints;
@@ -54,12 +56,16 @@
 + (AWSRegion *)regionWithLocation:(NSString *)theLocation;
 + (AWSRegion *)regionWithS3Endpoint:(NSURL *)theEndpoint;
 + (AWSRegion *)usEast1;
++ (AWSRegion *)usEast2;
 + (AWSRegion *)usWest1;
 + (AWSRegion *)usWest2;
 + (AWSRegion *)euWest1;
++ (AWSRegion *)euCentral1;
 + (AWSRegion *)apSoutheast1;
 + (AWSRegion *)apSoutheast2;
 + (AWSRegion *)apNortheast1;
++ (AWSRegion *)apNortheast2;
++ (AWSRegion *)apSouth1;
 + (AWSRegion *)saEast1;
 
 - (NSString *)regionName;

cocoastack/aws/AWSRegion.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #import "AWSRegion.h"
 #import "RegexKitLite.h"
 
@@ -39,12 +41,16 @@
 + (NSArray *)allRegions {
     return [NSArray arrayWithObjects:
             [AWSRegion usEast1],
+            [AWSRegion usEast2],
             [AWSRegion usWest1],
             [AWSRegion usWest2],
             [AWSRegion euWest1],
+            [AWSRegion euCentral1],
             [AWSRegion apSoutheast1],
             [AWSRegion apSoutheast2],
             [AWSRegion apNortheast1],
+            [AWSRegion apNortheast2],
+            [AWSRegion apSouth1],
             [AWSRegion saEast1],
             nil];
 }
@@ -103,14 +109,29 @@
                             s3LocationConstraints:nil
                                        s3Hostname:@"s3.amazonaws.com"
                                       displayName:@"US East (Northern Virginia)"
-                                 shortDisplayName:@"N. Virginia"
+                                 shortDisplayName:@"Virginia"
-               s3StorageDollarsPerGBMonthStandard:.030
+               s3StorageDollarsPerGBMonthStandard:.023
-                    s3StorageDollarsPerGBMonthRRS:.024
+                    s3StorageDollarsPerGBMonthRRS:.0125
                              s3UploadDollarsPerGB:.005
-                    s3DataTransferOutDollarsPerGB:.12
+                    s3DataTransferOutDollarsPerGB:.09
-                  glacierStorageDollarsPerGBMonth:.01
+                  glacierStorageDollarsPerGBMonth:.004
                         glacierUploadDollarsPerGB:.05
-               glacierDataTransferOutDollarsPerGB:.12
+               glacierDataTransferOutDollarsPerGB:.09
+                                  supportsGlacier:YES] autorelease];
+}
++ (AWSRegion *)usEast2 {
+    return [[[AWSRegion alloc] initWithRegionName:@"us-east-2"
+                            s3LocationConstraints:[NSArray arrayWithObject:@"us-east-2"]
+                                       s3Hostname:@"s3-us-east-2.amazonaws.com"
+                                      displayName:@"US East (Ohio)"
+                                 shortDisplayName:@"Ohio"
+               s3StorageDollarsPerGBMonthStandard:.023
+                    s3StorageDollarsPerGBMonthRRS:.0125
+                             s3UploadDollarsPerGB:.005
+                    s3DataTransferOutDollarsPerGB:.09
+                  glacierStorageDollarsPerGBMonth:.004
+                        glacierUploadDollarsPerGB:.05
+               glacierDataTransferOutDollarsPerGB:.09
                                   supportsGlacier:YES] autorelease];
 }
 + (AWSRegion *)usWest1 {
@@ -118,14 +139,14 @@
                             s3LocationConstraints:[NSArray arrayWithObject:@"us-west-1"]
                                        s3Hostname:@"s3-us-west-1.amazonaws.com"
                                       displayName:@"US West (Northern California)"
-                                 shortDisplayName:@"N. California"
+                                 shortDisplayName:@"California"
-               s3StorageDollarsPerGBMonthStandard:.033
+               s3StorageDollarsPerGBMonthStandard:.026
-                    s3StorageDollarsPerGBMonthRRS:.0264
+                    s3StorageDollarsPerGBMonthRRS:.019
                              s3UploadDollarsPerGB:.0055
-                    s3DataTransferOutDollarsPerGB:.12
+                    s3DataTransferOutDollarsPerGB:.09
-                  glacierStorageDollarsPerGBMonth:.011
+                  glacierStorageDollarsPerGBMonth:.005
                         glacierUploadDollarsPerGB:.055
-               glacierDataTransferOutDollarsPerGB:.12
+               glacierDataTransferOutDollarsPerGB:.09
                                   supportsGlacier:YES] autorelease];
 }
 + (AWSRegion *)usWest2 {
@@ -134,13 +155,13 @@
                                        s3Hostname:@"s3-us-west-2.amazonaws.com"
                                       displayName:@"US West (Oregon)"
                                  shortDisplayName:@"Oregon"
-               s3StorageDollarsPerGBMonthStandard:.030
+               s3StorageDollarsPerGBMonthStandard:.023
-                    s3StorageDollarsPerGBMonthRRS:.024
+                    s3StorageDollarsPerGBMonthRRS:.0125
                              s3UploadDollarsPerGB:.005
-                    s3DataTransferOutDollarsPerGB:.12
+                    s3DataTransferOutDollarsPerGB:.09
-                  glacierStorageDollarsPerGBMonth:.01
+                  glacierStorageDollarsPerGBMonth:.004
                         glacierUploadDollarsPerGB:.05
-               glacierDataTransferOutDollarsPerGB:.12
+               glacierDataTransferOutDollarsPerGB:.09
                                   supportsGlacier:YES] autorelease];
 }
 + (AWSRegion *)euWest1 {
@@ -149,13 +170,28 @@
                                        s3Hostname:@"s3-eu-west-1.amazonaws.com"
                                       displayName:@"EU (Ireland)"
                                  shortDisplayName:@"Ireland"
-               s3StorageDollarsPerGBMonthStandard:.030
+               s3StorageDollarsPerGBMonthStandard:.023
-                    s3StorageDollarsPerGBMonthRRS:.024
+                    s3StorageDollarsPerGBMonthRRS:.0125
                              s3UploadDollarsPerGB:.005
-                    s3DataTransferOutDollarsPerGB:.12
+                    s3DataTransferOutDollarsPerGB:.09
-                  glacierStorageDollarsPerGBMonth:.011
+                  glacierStorageDollarsPerGBMonth:.004
                         glacierUploadDollarsPerGB:.055
-               glacierDataTransferOutDollarsPerGB:.12
+               glacierDataTransferOutDollarsPerGB:.09
+                                  supportsGlacier:YES] autorelease];
+}
++ (AWSRegion *)euCentral1 {
+    return [[[AWSRegion alloc] initWithRegionName:@"eu-central-1"
+                            s3LocationConstraints:[NSArray arrayWithObjects:@"eu-central-1", nil]
+                                       s3Hostname:@"s3.eu-central-1.amazonaws.com"
+                                      displayName:@"EU (Frankfurt)"
+                                 shortDisplayName:@"Frankfurt"
+               s3StorageDollarsPerGBMonthStandard:.0245
+                    s3StorageDollarsPerGBMonthRRS:.0135
+                             s3UploadDollarsPerGB:.0054
+                    s3DataTransferOutDollarsPerGB:.09
+                  glacierStorageDollarsPerGBMonth:.0045
+                        glacierUploadDollarsPerGB:.06
+               glacierDataTransferOutDollarsPerGB:.09
                                   supportsGlacier:YES] autorelease];
 }
 + (AWSRegion *)apSoutheast1 {
@@ -164,10 +200,10 @@
                                        s3Hostname:@"s3-ap-southeast-1.amazonaws.com"
                                       displayName:@"Asia Pacific (Singapore)"
                                  shortDisplayName:@"Singapore"
-               s3StorageDollarsPerGBMonthStandard:.030
+               s3StorageDollarsPerGBMonthStandard:.025
-                    s3StorageDollarsPerGBMonthRRS:.024
+                    s3StorageDollarsPerGBMonthRRS:.02
                              s3UploadDollarsPerGB:.005
-                    s3DataTransferOutDollarsPerGB:.19
+                    s3DataTransferOutDollarsPerGB:.12
                   glacierStorageDollarsPerGBMonth:0
                         glacierUploadDollarsPerGB:0
                glacierDataTransferOutDollarsPerGB:0
@@ -179,13 +215,13 @@
                                        s3Hostname:@"s3-ap-southeast-2.amazonaws.com"
                                       displayName:@"Asia Pacific (Sydney)"
                                  shortDisplayName:@"Sydney"
-               s3StorageDollarsPerGBMonthStandard:.033
+               s3StorageDollarsPerGBMonthStandard:.025
-                    s3StorageDollarsPerGBMonthRRS:.0264
+                    s3StorageDollarsPerGBMonthRRS:.019
                              s3UploadDollarsPerGB:.0055
-                    s3DataTransferOutDollarsPerGB:.19
+                    s3DataTransferOutDollarsPerGB:.14
-                  glacierStorageDollarsPerGBMonth:.012
+                  glacierStorageDollarsPerGBMonth:.005
                         glacierUploadDollarsPerGB:.06
-               glacierDataTransferOutDollarsPerGB:.19
+               glacierDataTransferOutDollarsPerGB:.14
                                   supportsGlacier:YES] autorelease];
 }
 + (AWSRegion *)apNortheast1 {
@@ -194,13 +230,43 @@
                                        s3Hostname:@"s3-ap-northeast-1.amazonaws.com"
                                       displayName:@"Asia Pacific (Tokyo)"
                                  shortDisplayName:@"Tokyo"
-               s3StorageDollarsPerGBMonthStandard:.033
+               s3StorageDollarsPerGBMonthStandard:.025
-                    s3StorageDollarsPerGBMonthRRS:.0264
+                    s3StorageDollarsPerGBMonthRRS:.019
                              s3UploadDollarsPerGB:.005
-                    s3DataTransferOutDollarsPerGB:.201
+                    s3DataTransferOutDollarsPerGB:.14
-                  glacierStorageDollarsPerGBMonth:.012
+                  glacierStorageDollarsPerGBMonth:.005
                         glacierUploadDollarsPerGB:.06
-               glacierDataTransferOutDollarsPerGB:.201
+               glacierDataTransferOutDollarsPerGB:.14
+                                  supportsGlacier:YES] autorelease];
+}
++ (AWSRegion *)apNortheast2 {
+    return [[[AWSRegion alloc] initWithRegionName:@"ap-northeast-2"
+                            s3LocationConstraints:[NSArray arrayWithObject:@"ap-northeast-2"]
+                                       s3Hostname:@"s3-ap-northeast-2.amazonaws.com"
+                                      displayName:@"Asia Pacific (Seoul)"
+                                 shortDisplayName:@"Seoul"
+               s3StorageDollarsPerGBMonthStandard:.025
+                    s3StorageDollarsPerGBMonthRRS:.018
+                             s3UploadDollarsPerGB:.0045
+                    s3DataTransferOutDollarsPerGB:.126
+                  glacierStorageDollarsPerGBMonth:.005
+                        glacierUploadDollarsPerGB:.06
+               glacierDataTransferOutDollarsPerGB:.126
+                                  supportsGlacier:YES] autorelease];
+}
++ (AWSRegion *)apSouth1 {
+    return [[[AWSRegion alloc] initWithRegionName:@"ap-south-1"
+                            s3LocationConstraints:[NSArray arrayWithObject:@"ap-south-1"]
+                                       s3Hostname:@"s3-ap-south-1.amazonaws.com"
+                                      displayName:@"Asia Pacific (Mumbai)"
+                                 shortDisplayName:@"Mumbai"
+               s3StorageDollarsPerGBMonthStandard:.025
+                    s3StorageDollarsPerGBMonthRRS:.019
+                             s3UploadDollarsPerGB:.0045
+                    s3DataTransferOutDollarsPerGB:.126
+                  glacierStorageDollarsPerGBMonth:.005
+                        glacierUploadDollarsPerGB:.06
+               glacierDataTransferOutDollarsPerGB:.126
                                   supportsGlacier:YES] autorelease];
 }
 + (AWSRegion *)saEast1 {
@@ -209,8 +275,8 @@
                                        s3Hostname:@"s3-sa-east-1.amazonaws.com"
                                       displayName:@"South America (Sao Paulo)"
                                  shortDisplayName:@"Sao Paulo"
-               s3StorageDollarsPerGBMonthStandard:.0408
+               s3StorageDollarsPerGBMonthStandard:.0405
-                    s3StorageDollarsPerGBMonthRRS:.0326
+                    s3StorageDollarsPerGBMonthRRS:.026
                              s3UploadDollarsPerGB:.007
                     s3DataTransferOutDollarsPerGB:.25
                   glacierStorageDollarsPerGBMonth:0

cocoastack/aws/SignatureV2Provider.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,6 +31,8 @@
  */
 
 
+
+
 @interface SignatureV2Provider : NSObject {
     NSData *secretKeyData;
 }

cocoastack/aws/SignatureV2Provider.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #include <CommonCrypto/CommonHMAC.h>
 #import "SignatureV2Provider.h"
 #import "NSData-Base64Extensions.h"

cocoastack/crypto/CryptoKey.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #ifdef USE_OPENSSL
 #import "OpenSSLCryptoKey.h"
 #else
@@ -49,5 +51,7 @@
 - (id)initLegacyWithPassword:(NSString *)thePassword error:(NSError **)error;
 
 - (NSData *)encrypt:(NSData *)plainData error:(NSError **)error;
+- (BOOL)encrypt:(NSData *)plainData intoBuffer:(NSMutableData *)theOutBuffer error:(NSError **)error;
 - (NSData *)decrypt:(NSData *)encrypted error:(NSError **)error;
+- (BOOL)decrypt:(NSData *)encrypted intoBuffer:(NSMutableData *)theOutBuffer error:(NSError **)error;
 @end

cocoastack/crypto/CryptoKey.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #import "CryptoKey.h"
 
 #ifdef USE_OPENSSL
@@ -57,10 +59,10 @@
 
 #ifdef USE_OPENSSL
         cryptoKey = [[OpenSSLCryptoKey alloc] initWithPassword:thePassword salt:theSalt error:error];
-        HSLogDebug(@"using OpenSSL");
+//        HSLogDebug(@"using OpenSSL");
 #else
         cryptoKey = [[CCCryptoKey alloc] initWithPassword:thePassword salt:theSalt error:error];
-        HSLogDebug(@"using CommonCrypto");
+//        HSLogDebug(@"using CommonCrypto");
 #endif
         if (cryptoKey == nil) {
             [self release];
@@ -96,7 +98,13 @@
 - (NSData *)encrypt:(NSData *)plainData error:(NSError **)error {
     return [cryptoKey encrypt:plainData error:error];
 }
+- (BOOL)encrypt:(NSData *)plainData intoBuffer:(NSMutableData *)theOutBuffer error:(NSError **)error {
+    return [cryptoKey encrypt:plainData intoBuffer:theOutBuffer error:error];
+}
 - (NSData *)decrypt:(NSData *)encrypted error:(NSError **)error {
     return [cryptoKey decrypt:encrypted error:error];
 }
+- (BOOL)decrypt:(NSData *)encrypted intoBuffer:(NSMutableData *)theOutBuffer error:(NSError **)error {
+    return [cryptoKey decrypt:encrypted intoBuffer:theOutBuffer error:error];
+}
 @end

cocoastack/crypto/HMACSHA256.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,15 +30,12 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#import "RemoteFS.h"
-@class Target;
-@class S3Service;
 
 
-@interface S3RemoteFS : NSObject <RemoteFS> {
+
-    Target *target;
+@interface HMACSHA256 : NSObject {
-    S3Service *s3;
+    
 }
++ (NSData *)digestForKey:(NSData *)theKey data:(NSData *)theData;
 
-- (id)initWithTarget:(Target *)theTarget;
 @end

cocoastack/crypto/HMACSHA256.m

@@ -0,0 +1,45 @@
+/*
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
+ 
+ All rights reserved.
+ 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ 
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 
+ * Neither the names of PhotoMinds LLC or Haystack Software, nor the names of
+ their contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+#include <CommonCrypto/CommonHMAC.h>
+#import "HMACSHA256.h"
+
+@implementation HMACSHA256
++ (NSData *)digestForKey:(NSData *)theKey data:(NSData *)theData {
+    unsigned char digest[CC_SHA256_DIGEST_LENGTH];
+    CCHmac(kCCHmacAlgSHA256, [theKey bytes], [theKey length], [theData bytes], [theData length], digest);
+    NSData *hmacSHA1 = [[[NSData alloc] initWithBytes:digest length:CC_SHA256_DIGEST_LENGTH] autorelease];
+    return hmacSHA1;
+}
+@end

cocoastack/crypto/MD5Hash.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,6 +31,9 @@
  */
 
 
+
+
 @interface MD5Hash : NSObject
 + (NSString *)hashDataBase64Encode:(NSData *)theData;
++ (NSString *)hashData:(NSData *)data;
 @end

cocoastack/crypto/MD5Hash.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #import <CommonCrypto/CommonDigest.h>
 #import "MD5Hash.h"
 #import "NSString_extra.h"
@@ -46,4 +48,12 @@
     NSData *digestData = [NSData dataWithBytes:digest length:CC_MD5_DIGEST_LENGTH];
     return [digestData encodeBase64];
 }
++ (NSString *)hashData:(NSData *)data {
+    unsigned char digest[CC_MD5_DIGEST_LENGTH];
+    memset(digest, 0, CC_MD5_DIGEST_LENGTH);
+    if (CC_MD5([data bytes], (CC_LONG)[data length], digest) == NULL) {
+        HSLogError(@"CC_MD5 failed!");
+    }
+    return [NSString hexStringWithBytes:digest length:CC_MD5_DIGEST_LENGTH];
+}
 @end

cocoastack/crypto/OpenSSL.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #import <openssl/ssl.h>
 
 @interface OpenSSL : NSObject {

cocoastack/crypto/OpenSSL.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #ifdef USE_OPENSSL
 
 
@@ -74,7 +76,6 @@ static SSL_CTX *ctx;
         if ([msg length] > 0) {
             [msg appendString:@"; "];
         }
-        HSLogTrace(@"%s", ERR_error_string(err, NULL));
         [msg appendFormat:@"%s", ERR_reason_error_string(err)];
     }
     if ([msg length] == 0) {
@@ -87,4 +88,4 @@ static SSL_CTX *ctx;
 @end
 
 
-#endif /* USE_OPENSSL */
+#endif /* USE_OPENSSL */

cocoastack/crypto/OpenSSLCryptoKey.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #ifdef USE_OPENSSL
 
 #include <openssl/evp.h>
@@ -44,7 +46,9 @@
 - (id)initLegacyWithPassword:(NSString *)thePassword error:(NSError **)error;
 
 - (NSData *)encrypt:(NSData *)plainData error:(NSError **)error;
+- (BOOL)encrypt:(NSData *)plainData intoBuffer:(NSMutableData *)theOutBuffer error:(NSError **)error;
 - (NSData *)decrypt:(NSData *)encrypted error:(NSError **)error;
+- (BOOL)decrypt:(NSData *)encrypted intoBuffer:(NSMutableData *)theOutBuffer error:(NSError **)error;
 
 @end
 

cocoastack/crypto/OpenSSLCryptoKey.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #ifdef USE_OPENSSL
 
 
@@ -127,47 +129,63 @@
 }
 
 - (NSData *)encrypt:(NSData *)plainData error:(NSError **)error {
-    if ([plainData length] == 0) {
+    NSMutableData *outBuffer = [NSMutableData data];
-        return [NSData data];
+    if (![self encrypt:plainData intoBuffer:outBuffer error:error]) {
+        return nil;
     }
+    return outBuffer;
+}
+- (BOOL)encrypt:(NSData *)plainData intoBuffer:(NSMutableData *)theOutBuffer error:(NSError **)error {
+    [theOutBuffer setLength:0];
+    if ([plainData length] == 0) {
+        return YES;
+    }
+    
     EVP_CIPHER_CTX cipherContext;
     EVP_CIPHER_CTX_init(&cipherContext);
     if (!EVP_EncryptInit(&cipherContext, cipher, evpKey, iv)) {
         SETNSERROR([CryptoKey errorDomain], -1, @"EVP_EncryptInit: %@", [OpenSSL errorMessage]);
         EVP_CIPHER_CTX_cleanup(&cipherContext);
-        return nil;
+        return NO;
     }
     
     // Need room for data + cipher block size - 1.
-    unsigned char *outbuf = (unsigned char *)malloc([plainData length] + EVP_CIPHER_CTX_block_size(&cipherContext) - 1);
+    [theOutBuffer setLength:([plainData length] + EVP_CIPHER_CTX_block_size(&cipherContext))];
+    unsigned char *outbuf = (unsigned char *)[theOutBuffer mutableBytes];
     
     int outlen = 0;
     if (!EVP_EncryptUpdate(&cipherContext, outbuf, &outlen, [plainData bytes], (int)[plainData length])) {
         SETNSERROR([CryptoKey errorDomain], -1, @"EVP_EncryptUpdate: %@",  [OpenSSL errorMessage]);
-        free(outbuf);
         EVP_CIPHER_CTX_cleanup(&cipherContext);
-        return nil;
+        return NO;
     }
     
     int extralen = 0;
     if (!EVP_EncryptFinal(&cipherContext, outbuf + outlen, &extralen)) {
         SETNSERROR([CryptoKey errorDomain], -1, @"EVP_EncryptFinal: %@",  [OpenSSL errorMessage]);
-        free(outbuf);
         EVP_CIPHER_CTX_cleanup(&cipherContext);
-        return nil;
+        return NO;
     }
     EVP_CIPHER_CTX_cleanup(&cipherContext);
     
-    NSData *ret = [[[NSData alloc] initWithBytesNoCopy:outbuf length:(outlen + extralen)] autorelease];
+    [theOutBuffer setLength:(outlen + extralen)];
-    return ret;
+    return YES;
 }
 - (NSData *)decrypt:(NSData *)encrypted error:(NSError **)error {
-    if (encrypted == nil) {
+    NSMutableData *outBuffer = [NSMutableData data];
-        SETNSERROR([CryptoKey errorDomain], -1, @"decrypt: nil input NSData");
+    if (![self decrypt:encrypted intoBuffer:outBuffer error:error]) {
         return nil;
     }
+    return outBuffer;
+}
+- (BOOL)decrypt:(NSData *)encrypted intoBuffer:(NSMutableData *)theOutBuffer error:(NSError **)error {
+    if (encrypted == nil) {
+        SETNSERROR([CryptoKey errorDomain], -1, @"decrypt: nil input NSData");
+        return NO;
+    }
     if ([encrypted length] == 0) {
-        return [NSData data];
+        [theOutBuffer setLength:0];
+        return YES;
     }
     
     int inlen = (int)[encrypted length];
@@ -178,30 +196,28 @@
     if (!EVP_DecryptInit(&cipherContext, cipher, evpKey, iv)) {
         SETNSERROR([CryptoKey errorDomain], -1, @"EVP_DecryptInit: %@", [OpenSSL errorMessage]);
         EVP_CIPHER_CTX_cleanup(&cipherContext);
-        return nil;
+        return NO;
     }
     
-    unsigned char *outbuf = (unsigned char *)malloc(inlen + EVP_CIPHER_CTX_block_size(&cipherContext));
+    [theOutBuffer setLength:(inlen + EVP_CIPHER_CTX_block_size(&cipherContext))];
-    
+    unsigned char *outbuf = (unsigned char *)[theOutBuffer mutableBytes];
     int outlen = 0;
     if (!EVP_DecryptUpdate(&cipherContext, outbuf, &outlen, input, inlen)) {
         SETNSERROR([CryptoKey errorDomain], -1, @"EVP_DecryptUpdate: %@", [OpenSSL errorMessage]);
-        free(outbuf);
         EVP_CIPHER_CTX_cleanup(&cipherContext);
-        return nil;
+        return NO;
     }
     
     int extralen = 0;
     if (!EVP_DecryptFinal(&cipherContext, outbuf + outlen, &extralen)) {
         SETNSERROR([CryptoKey errorDomain], -1, @"EVP_DecryptFinal: %@", [OpenSSL errorMessage]);
-        free(outbuf);
         EVP_CIPHER_CTX_cleanup(&cipherContext);
-        return nil;
+        return NO;
     }
     
     EVP_CIPHER_CTX_cleanup(&cipherContext);
-    NSData *ret = [[[NSData alloc] initWithBytesNoCopy:outbuf length:(outlen + extralen)] autorelease];
+    [theOutBuffer setLength:(outlen + extralen)];
-    return ret;
+    return YES;
 }
 @end
 

cocoastack/crypto/SHA1Hash.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,6 +31,7 @@
  */
 
 
+
 #import "InputStream.h"
 
 @interface SHA1Hash : NSObject {

cocoastack/crypto/SHA1Hash.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #import <CommonCrypto/CommonDigest.h>
 #include <sys/stat.h>
 #import "SHA1Hash.h"

cocoastack/crypto/SHA256Hash.h

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -31,6 +31,8 @@
  */
 
 
+
+
 @interface SHA256Hash : NSObject {
     
 }

cocoastack/crypto/SHA256Hash.m

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2009-2014, Stefan Reitshamer http://www.haystacksoftware.com
+ Copyright (c) 2009-2017, Haystack Software LLC https://www.arqbackup.com
  
  All rights reserved.
  
@@ -30,6 +30,8 @@
  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+
+
 #import "SHA256Hash.h"
 #import <CommonCrypto/CommonDigest.h>