From 20004756bcc27c8a9b4f288741844457c8accdd3 Mon Sep 17 00:00:00 2001 From: Stefan Reitshamer Date: Mon, 28 Jul 2014 16:11:39 -0400 Subject: [PATCH] Implemented restore from S3/Glacier and from (legacy) Glacier. --- ArqRestoreCommand.h | 4 +- ArqRestoreCommand.m | 179 +++- arq_restore.m | 6 +- arq_restore.xcodeproj/project.pbxproj | 74 ++ commonrestore/GlacierRequestItem.h | 26 + commonrestore/GlacierRequestItem.m | 106 ++ glacierrestore/GlacierPack.h | 38 + glacierrestore/GlacierPack.m | 121 +++ glacierrestore/GlacierPackIndex.h | 53 + glacierrestore/GlacierPackIndex.m | 405 ++++++++ glacierrestore/GlacierPackSet.h | 47 + glacierrestore/GlacierPackSet.m | 134 +++ glacierrestore/GlacierRestorer.h | 69 ++ glacierrestore/GlacierRestorer.m | 964 +++++++++++++++++++ glacierrestore/GlacierRestorerDelegate.h | 27 + glacierrestore/GlacierRestorerParamSet.h | 58 ++ glacierrestore/GlacierRestorerParamSet.m | 79 ++ repo/BinarySHA1.h | 14 + repo/BinarySHA1.m | 25 + s3glacierrestore/S3GlacierRestorer.h | 50 + s3glacierrestore/S3GlacierRestorer.m | 445 +++++++++ s3glacierrestore/S3GlacierRestorerDelegate.h | 28 + s3glacierrestore/S3GlacierRestorerParamSet.h | 61 ++ s3glacierrestore/S3GlacierRestorerParamSet.m | 79 ++ 24 files changed, 3049 insertions(+), 43 deletions(-) create mode 100644 commonrestore/GlacierRequestItem.h create mode 100644 commonrestore/GlacierRequestItem.m create mode 100644 glacierrestore/GlacierPack.h create mode 100644 glacierrestore/GlacierPack.m create mode 100644 glacierrestore/GlacierPackIndex.h create mode 100644 glacierrestore/GlacierPackIndex.m create mode 100644 glacierrestore/GlacierPackSet.h create mode 100644 glacierrestore/GlacierPackSet.m create mode 100644 glacierrestore/GlacierRestorer.h create mode 100644 glacierrestore/GlacierRestorer.m create mode 100644 glacierrestore/GlacierRestorerDelegate.h create mode 100644 glacierrestore/GlacierRestorerParamSet.h create mode 100644 glacierrestore/GlacierRestorerParamSet.m create mode 100644 repo/BinarySHA1.h create mode 100644 repo/BinarySHA1.m create mode 100644 s3glacierrestore/S3GlacierRestorer.h create mode 100644 s3glacierrestore/S3GlacierRestorer.m create mode 100644 s3glacierrestore/S3GlacierRestorerDelegate.h create mode 100644 s3glacierrestore/S3GlacierRestorerParamSet.h create mode 100644 s3glacierrestore/S3GlacierRestorerParamSet.m diff --git a/ArqRestoreCommand.h b/ArqRestoreCommand.h index 1a33f16..273fc72 100644 --- a/ArqRestoreCommand.h +++ b/ArqRestoreCommand.h @@ -7,10 +7,12 @@ // #import "S3RestorerDelegate.h" +#import "S3GlacierRestorerDelegate.h" +#import "GlacierRestorerDelegate.h" @class Target; -@interface ArqRestoreCommand : NSObject { +@interface ArqRestoreCommand : NSObject { Target *target; } diff --git a/ArqRestoreCommand.m b/ArqRestoreCommand.m index a166a1e..a4be119 100644 --- a/ArqRestoreCommand.m +++ b/ArqRestoreCommand.m @@ -19,6 +19,11 @@ #import "Commit.h" #import "BlobKey.h" #import "S3Restorer.h" +#import "S3GlacierRestorerParamSet.h" +#import "S3GlacierRestorer.h" +#import "GlacierRestorerParamSet.h" +#import "GlacierRestorer.h" +#import "S3AuthorizationProvider.h" @implementation ArqRestoreCommand @@ -65,11 +70,11 @@ } targetParamsIndex += 2; } else if ([cmd isEqualToString:@"restore"]) { - if ((argc - targetParamsIndex) < 3) { + if ((argc - targetParamsIndex) < 4) { SETNSERROR([self errorDomain], ERROR_USAGE, @"missing arguments"); return NO; } - targetParamsIndex += 3; + targetParamsIndex += 4; } else { SETNSERROR([self errorDomain], ERROR_USAGE, @"unknown command: %@", cmd); return NO; @@ -93,7 +98,7 @@ return NO; } } else if ([cmd isEqualToString:@"restore"]) { - if (![self restoreComputerUUID:[args objectAtIndex:index+1] bucketUUID:[args objectAtIndex:index+3] encryptionPassword:[args objectAtIndex:index+2] error:error]) { + if (![self restoreComputerUUID:[args objectAtIndex:index+1] bucketUUID:[args objectAtIndex:index+3] encryptionPassword:[args objectAtIndex:index+2] restoreBytesPerSecond:[args objectAtIndex:index+3] error:error]) { return NO; } } else { @@ -201,7 +206,15 @@ } - (AWSRegion *)awsRegionForAccessKey:(NSString *)theAccessKey secretKey:(NSString *)theSecretKey bucketName:(NSString *)theBucketName error:(NSError **)error { - return nil; + S3AuthorizationProvider *sap = [[[S3AuthorizationProvider alloc] initWithAccessKey:theAccessKey secretKey:theSecretKey] autorelease]; + NSURL *endpoint = [[AWSRegion usEast1] s3EndpointWithSSL:YES]; + S3Service *s3 = [[[S3Service alloc] initWithS3AuthorizationProvider:sap endpoint:endpoint useAmazonRRS:NO] autorelease]; + + NSString *location = [s3 locationOfS3Bucket:theBucketName targetConnectionDelegate:nil error:error]; + if (location == nil) { + return nil; + } + return [AWSRegion regionWithLocation:location]; } - (BOOL)listComputers:(NSError **)error { @@ -236,20 +249,20 @@ } - (NSArray *)expandedTargetList:(NSError **)error { NSMutableArray *expandedTargetList = [NSMutableArray arrayWithObject:target]; - if ([target targetType] == kTargetAWS - || [target targetType] == kTargetDreamObjects - || [target targetType] == kTargetGoogleCloudStorage - || [target targetType] == kTargetGreenQloud - || [target targetType] == kTargetS3Compatible) { - NSError *myError = nil; - NSArray *targets = [self expandedTargetsForS3Target:target error:&myError]; - if (targets == nil) { - HSLogError(@"failed to expand target list for %@: %@", target, myError); - } else { - [expandedTargetList setArray:targets]; - HSLogDebug(@"expandedTargetList is now: %@", expandedTargetList); - } - } +// if ([target targetType] == kTargetAWS +// || [target targetType] == kTargetDreamObjects +// || [target targetType] == kTargetGoogleCloudStorage +// || [target targetType] == kTargetGreenQloud +// || [target targetType] == kTargetS3Compatible) { +// NSError *myError = nil; +// NSArray *targets = [self expandedTargetsForS3Target:target error:&myError]; +// if (targets == nil) { +// HSLogError(@"failed to expand target list for %@: %@", target, myError); +// } else { +// [expandedTargetList setArray:targets]; +// HSLogDebug(@"expandedTargetList is now: %@", expandedTargetList); +// } +// } return expandedTargetList; } - (NSArray *)expandedTargetsForS3Target:(Target *)theTarget error:(NSError **)error { @@ -307,7 +320,7 @@ return YES; } -- (BOOL)restoreComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID encryptionPassword:(NSString *)theEncryptionPassword error:(NSError **)error { +- (BOOL)restoreComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID encryptionPassword:(NSString *)theEncryptionPassword restoreBytesPerSecond:(NSString *)theRestoreBytesPerSecond error:(NSError **)error { Bucket *myBucket = nil; NSArray *expandedTargetList = [self expandedTargetList:error]; if (expandedTargetList == nil) { @@ -363,28 +376,51 @@ AWSRegion *region = [AWSRegion regionWithS3Endpoint:[target endpoint]]; BOOL isGlacierDestination = [region supportsGlacier]; if ([myBucket storageType] == StorageTypeGlacier && isGlacierDestination) { -// [[[GlacierRestoreController alloc] initWithAppConfig:appConfig -// doChownsAbove499:NO -// destinationPath:destination -// displayBucket:sourceOutlineController.selectedDisplayBucket -// displayCommit:sourceOutlineController.selectedDisplayCommit -// displayNode:selectedNode -// mainWindow:mainWindow] autorelease]; + int bytesPerSecond = [theRestoreBytesPerSecond intValue]; + if (bytesPerSecond == 0) { + SETNSERROR([self errorDomain], -1, @"invalid bytes_per_second %@", theRestoreBytesPerSecond); + return NO; + } + + GlacierRestorerParamSet *paramSet = [[[GlacierRestorerParamSet alloc] initWithBucket:myBucket + encryptionPassword:theEncryptionPassword + downloadBytesPerSecond:bytesPerSecond + commitBlobKey:commitBlobKey + rootItemName:[[myBucket localPath] lastPathComponent] + treeVersion:CURRENT_TREE_VERSION + treeIsCompressed:[[commit treeBlobKey] compressed] + treeBlobKey:[commit treeBlobKey] + nodeName:nil targetUID:getuid() + targetGID:getgid() + useTargetUIDAndGID:YES + destinationPath:destinationPath + logLevel:global_hslog_level] autorelease]; + [[[GlacierRestorer alloc] initWithGlacierRestorerParamSet:paramSet delegate:self] autorelease]; + } else if ([myBucket storageType] == StorageTypeS3Glacier && isGlacierDestination) { -// [[[S3GlacierRestoreSetupController alloc] initWithLocalComputerUUID:[appConfig computerUUIDForTargetUUID:[target targetUUID]] -// doChownsAbove499:NO -// destinationPath:destination -// displayBucket:sourceOutlineController.selectedDisplayBucket -// displayCommit:sourceOutlineController.selectedDisplayCommit -// displayNode:selectedNode -// mainWindow:mainWindow] autorelease]; + int bytesPerSecond = [theRestoreBytesPerSecond intValue]; + if (bytesPerSecond == 0) { + SETNSERROR([self errorDomain], -1, @"invalid bytes_per_second %@", theRestoreBytesPerSecond); + return NO; + } + + S3GlacierRestorerParamSet *paramSet = [[[S3GlacierRestorerParamSet alloc] initWithBucket:myBucket + encryptionPassword:theEncryptionPassword + downloadBytesPerSecond:bytesPerSecond + commitBlobKey:commitBlobKey + rootItemName:[[myBucket localPath] lastPathComponent] + treeVersion:CURRENT_TREE_VERSION + treeIsCompressed:[[commit treeBlobKey] compressed] + treeBlobKey:[commit treeBlobKey] + nodeName:nil + targetUID:getuid() + targetGID:getgid() + useTargetUIDAndGID:YES + destinationPath:destinationPath + logLevel:global_hslog_level] autorelease]; + [[[S3GlacierRestorer alloc] initWithS3GlacierRestorerParamSet:paramSet delegate:self] autorelease]; + } else { -// [[[S3RestoreController alloc] initWithAppConfig:appConfig -// doChownsAbove499:doChowns -// destinationPath:destination -// displayBucket:sourceOutlineController.selectedDisplayBucket -// displayCommit:sourceOutlineController.selectedDisplayCommit - // displayNode:selectedNode] autorelease]; S3RestorerParamSet *paramSet = [[[S3RestorerParamSet alloc] initWithBucket:myBucket encryptionPassword:theEncryptionPassword commitBlobKey:commitBlobKey @@ -429,4 +465,69 @@ printf("failed: %s\n", [[error localizedDescription] UTF8String]); return NO; } + + +#pragma mark S3GlacierRestorerDelegate +- (BOOL)s3GlacierRestorerMessageDidChange:(NSString *)message { + printf("status: %s\n", [message UTF8String]); + return NO; +} +- (BOOL)s3GlacierRestorerBytesRequestedDidChange:(NSNumber *)theRequested { + return NO; +} +- (BOOL)s3GlacierRestorerTotalBytesToRequestDidChange:(NSNumber *)theMaxRequested { + return NO; +} +- (BOOL)s3GlacierRestorerDidFinishRequesting { + return NO; +} +- (BOOL)s3GlacierRestorerBytesTransferredDidChange:(NSNumber *)theTransferred { + return NO; +} +- (BOOL)s3GlacierRestorerTotalBytesToTransferDidChange:(NSNumber *)theTotal { + return NO; +} +- (BOOL)s3GlacierRestorerErrorMessage:(NSString *)theErrorMessage didOccurForPath:(NSString *)thePath { + printf("%s error: %s\n", [thePath UTF8String], [theErrorMessage UTF8String]); + return NO; +} +- (void)s3GlacierRestorerDidSucceed { +} +- (void)s3GlacierRestorerDidFail:(NSError *)error { + printf("failed: %s\n", [[error localizedDescription] UTF8String]); +} + + +#pragma mark GlacierRestorerDelegate +- (BOOL)glacierRestorerMessageDidChange:(NSString *)message { + printf("status: %s\n", [message UTF8String]); + return NO; +} +- (BOOL)glacierRestorerBytesRequestedDidChange:(NSNumber *)theRequested { + return NO; +} +- (BOOL)glacierRestorerTotalBytesToRequestDidChange:(NSNumber *)theMaxRequested { + return NO; +} +- (BOOL)glacierRestorerDidFinishRequesting { + return NO; +} +- (BOOL)glacierRestorerBytesTransferredDidChange:(NSNumber *)theTransferred { + return NO; +} +- (BOOL)glacierRestorerTotalBytesToTransferDidChange:(NSNumber *)theTotal { + return NO; +} +- (BOOL)glacierRestorerErrorMessage:(NSString *)theErrorMessage didOccurForPath:(NSString *)thePath { + printf("%s error: %s\n", [thePath UTF8String], [theErrorMessage UTF8String]); + return NO; +} +- (BOOL)glacierRestorerDidSucceed { + return NO; +} +- (BOOL)glacierRestorerDidFail:(NSError *)error { + printf("failed: %s\n", [[error localizedDescription] UTF8String]); + return NO; +} + @end diff --git a/arq_restore.m b/arq_restore.m index 374db93..8621415 100644 --- a/arq_restore.m +++ b/arq_restore.m @@ -37,9 +37,9 @@ static void printUsage(const char *exeName) { fprintf(stderr, "Usage:\n"); - fprintf(stderr, "\t%s [-l log_level] listcomputers \n", exeName); - fprintf(stderr, "\t%s [-l log_level] listfolders \n", exeName); - fprintf(stderr, "\t%s [-l log_level] restore \n", exeName); + fprintf(stderr, "\t%s [-l log_level] listcomputers \n", exeName); + fprintf(stderr, "\t%s [-l log_level] listfolders \n", exeName); + fprintf(stderr, "\t%s [-l log_level] restore \n", exeName); fprintf(stderr, "\t\ntarget_params by target type:\n"); fprintf(stderr, "\taws: access_key secret_key bucket_name\n"); fprintf(stderr, "\tsftp: hostname port path username password_or_keyfile [keyfile_passphrase]\n"); diff --git a/arq_restore.xcodeproj/project.pbxproj b/arq_restore.xcodeproj/project.pbxproj index b6ae033..5901325 100644 --- a/arq_restore.xcodeproj/project.pbxproj +++ b/arq_restore.xcodeproj/project.pbxproj @@ -189,6 +189,15 @@ F8F2D9851986D3C400997A15 /* XAttrSet.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9841986D3C400997A15 /* XAttrSet.m */; }; F8F2D9921986D4C700997A15 /* CalculateItem.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D98E1986D4C700997A15 /* CalculateItem.m */; }; F8F2D9931986D4C700997A15 /* RestoreItem.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9901986D4C700997A15 /* RestoreItem.m */; }; + F8F2D9981986DCCC00997A15 /* S3GlacierRestorerParamSet.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9971986DCCC00997A15 /* S3GlacierRestorerParamSet.m */; }; + F8F2D99C1986DDAD00997A15 /* S3GlacierRestorer.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D99A1986DDAD00997A15 /* S3GlacierRestorer.m */; }; + F8F2D99F1986DE1800997A15 /* GlacierRequestItem.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D99E1986DE1800997A15 /* GlacierRequestItem.m */; }; + F8F2D9A61986DE3B00997A15 /* GlacierPack.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9A11986DE3B00997A15 /* GlacierPack.m */; }; + F8F2D9A71986DE3B00997A15 /* GlacierPackIndex.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9A31986DE3B00997A15 /* GlacierPackIndex.m */; }; + F8F2D9A81986DE3B00997A15 /* GlacierPackSet.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9A51986DE3B00997A15 /* GlacierPackSet.m */; }; + F8F2D9AB1986DE4400997A15 /* GlacierRestorer.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9AA1986DE4400997A15 /* GlacierRestorer.m */; }; + F8F2D9AE1986DE8300997A15 /* BinarySHA1.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9AD1986DE8300997A15 /* BinarySHA1.m */; }; + F8F2D9B11986DF6B00997A15 /* GlacierRestorerParamSet.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9B01986DF6B00997A15 /* GlacierRestorerParamSet.m */; }; /* End PBXBuildFile section */ /* Begin PBXCopyFilesBuildPhase section */ @@ -579,6 +588,26 @@ F8F2D98F1986D4C700997A15 /* RestoreItem.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RestoreItem.h; sourceTree = ""; }; F8F2D9901986D4C700997A15 /* RestoreItem.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = RestoreItem.m; sourceTree = ""; }; F8F2D9911986D4C700997A15 /* Restorer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Restorer.h; sourceTree = ""; }; + F8F2D9961986DCCC00997A15 /* S3GlacierRestorerParamSet.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = S3GlacierRestorerParamSet.h; sourceTree = ""; }; + F8F2D9971986DCCC00997A15 /* S3GlacierRestorerParamSet.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = S3GlacierRestorerParamSet.m; sourceTree = ""; }; + F8F2D9991986DDAC00997A15 /* S3GlacierRestorer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = S3GlacierRestorer.h; sourceTree = ""; }; + F8F2D99A1986DDAD00997A15 /* S3GlacierRestorer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = S3GlacierRestorer.m; sourceTree = ""; }; + F8F2D99B1986DDAD00997A15 /* S3GlacierRestorerDelegate.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = S3GlacierRestorerDelegate.h; sourceTree = ""; }; + F8F2D99D1986DE1800997A15 /* GlacierRequestItem.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = GlacierRequestItem.h; sourceTree = ""; }; + F8F2D99E1986DE1800997A15 /* GlacierRequestItem.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GlacierRequestItem.m; sourceTree = ""; }; + F8F2D9A01986DE3B00997A15 /* GlacierPack.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = GlacierPack.h; sourceTree = ""; }; + F8F2D9A11986DE3B00997A15 /* GlacierPack.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GlacierPack.m; sourceTree = ""; }; + F8F2D9A21986DE3B00997A15 /* GlacierPackIndex.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = GlacierPackIndex.h; sourceTree = ""; }; + F8F2D9A31986DE3B00997A15 /* GlacierPackIndex.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GlacierPackIndex.m; sourceTree = ""; }; + F8F2D9A41986DE3B00997A15 /* GlacierPackSet.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = GlacierPackSet.h; sourceTree = ""; }; + F8F2D9A51986DE3B00997A15 /* GlacierPackSet.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GlacierPackSet.m; sourceTree = ""; }; + F8F2D9A91986DE4400997A15 /* GlacierRestorer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = GlacierRestorer.h; sourceTree = ""; }; + F8F2D9AA1986DE4400997A15 /* GlacierRestorer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GlacierRestorer.m; sourceTree = ""; }; + F8F2D9AC1986DE8300997A15 /* BinarySHA1.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BinarySHA1.h; sourceTree = ""; }; + F8F2D9AD1986DE8300997A15 /* BinarySHA1.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BinarySHA1.m; sourceTree = ""; }; + F8F2D9AF1986DF6B00997A15 /* GlacierRestorerParamSet.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = GlacierRestorerParamSet.h; sourceTree = ""; }; + F8F2D9B01986DF6B00997A15 /* GlacierRestorerParamSet.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GlacierRestorerParamSet.m; sourceTree = ""; }; + F8F2D9B21986DFF700997A15 /* GlacierRestorerDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GlacierRestorerDelegate.h; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -612,6 +641,8 @@ 08FB7795FE84155DC02AAC07 /* arq_restore */, F8F2D98C1986D49600997A15 /* commonrestore */, F8F2D9711986D00C00997A15 /* s3restore */, + F8F2D9951986DCBD00997A15 /* s3glacierrestore */, + F8F2D9941986DCB100997A15 /* glacierrestore */, 08FB779DFE84155DC02AAC07 /* External Frameworks and Libraries */, 1AB674ADFE9D54B511CA2CBB /* Products */, F89A204513FAE29E0071D321 /* libz.dylib */, @@ -1139,6 +1170,8 @@ F8F2D92B1986BA0700997A15 /* repo */ = { isa = PBXGroup; children = ( + F8F2D9AC1986DE8300997A15 /* BinarySHA1.h */, + F8F2D9AD1986DE8300997A15 /* BinarySHA1.m */, F8F2D92C1986BA1400997A15 /* Commit.h */, F8F2D92D1986BA1400997A15 /* Commit.m */, F8F2D92F1986BA2200997A15 /* CommitFailedFile.h */, @@ -1185,6 +1218,8 @@ children = ( F8F2D98D1986D4C700997A15 /* CalculateItem.h */, F8F2D98E1986D4C700997A15 /* CalculateItem.m */, + F8F2D99D1986DE1800997A15 /* GlacierRequestItem.h */, + F8F2D99E1986DE1800997A15 /* GlacierRequestItem.m */, F8F2D98F1986D4C700997A15 /* RestoreItem.h */, F8F2D9901986D4C700997A15 /* RestoreItem.m */, F8F2D9911986D4C700997A15 /* Restorer.h */, @@ -1192,6 +1227,36 @@ path = commonrestore; sourceTree = ""; }; + F8F2D9941986DCB100997A15 /* glacierrestore */ = { + isa = PBXGroup; + children = ( + F8F2D9A01986DE3B00997A15 /* GlacierPack.h */, + F8F2D9A11986DE3B00997A15 /* GlacierPack.m */, + F8F2D9A21986DE3B00997A15 /* GlacierPackIndex.h */, + F8F2D9A31986DE3B00997A15 /* GlacierPackIndex.m */, + F8F2D9A41986DE3B00997A15 /* GlacierPackSet.h */, + F8F2D9A51986DE3B00997A15 /* GlacierPackSet.m */, + F8F2D9A91986DE4400997A15 /* GlacierRestorer.h */, + F8F2D9AA1986DE4400997A15 /* GlacierRestorer.m */, + F8F2D9B21986DFF700997A15 /* GlacierRestorerDelegate.h */, + F8F2D9AF1986DF6B00997A15 /* GlacierRestorerParamSet.h */, + F8F2D9B01986DF6B00997A15 /* GlacierRestorerParamSet.m */, + ); + path = glacierrestore; + sourceTree = ""; + }; + F8F2D9951986DCBD00997A15 /* s3glacierrestore */ = { + isa = PBXGroup; + children = ( + F8F2D9991986DDAC00997A15 /* S3GlacierRestorer.h */, + F8F2D99A1986DDAD00997A15 /* S3GlacierRestorer.m */, + F8F2D99B1986DDAD00997A15 /* S3GlacierRestorerDelegate.h */, + F8F2D9961986DCCC00997A15 /* S3GlacierRestorerParamSet.h */, + F8F2D9971986DCCC00997A15 /* S3GlacierRestorerParamSet.m */, + ); + path = s3glacierrestore; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ @@ -1250,6 +1315,7 @@ F8F2D8821986B63400997A15 /* BaseTargetConnection.m in Sources */, F8F2D9931986D4C700997A15 /* RestoreItem.m in Sources */, F8F2D93D1986BA7900997A15 /* UserLibrary.m in Sources */, + F8F2D99F1986DE1800997A15 /* GlacierRequestItem.m in Sources */, F8F2D86E1986B5CC00997A15 /* SFTPTargetConnection.m in Sources */, F829522019868E26001DC91B /* GoogleDrive.m in Sources */, F8F2D9001986B78600997A15 /* BooleanNode.m in Sources */, @@ -1292,6 +1358,8 @@ F829DC10198691CB00D637E0 /* BooleanIO.m in Sources */, F8F2D9491986BAD200997A15 /* Repo.m in Sources */, F829522119868E26001DC91B /* GoogleDriveErrorResult.m in Sources */, + F8F2D99C1986DDAD00997A15 /* S3GlacierRestorer.m in Sources */, + F8F2D9B11986DF6B00997A15 /* GlacierRestorerParamSet.m in Sources */, F8295167198683D5001DC91B /* scp.c in Sources */, F8F2D9531986BE2A00997A15 /* FarkImpl.m in Sources */, F829DC241986924100D637E0 /* FDInputStream.m in Sources */, @@ -1312,7 +1380,9 @@ F8F2D9771986D32B00997A15 /* S3Restorer.m in Sources */, F8F2D8931986B66000997A15 /* GlacierJobLister.m in Sources */, F8F2D9031986B78600997A15 /* RealNode.m in Sources */, + F8F2D9A71986DE3B00997A15 /* GlacierPackIndex.m in Sources */, F8F2D87C1986B61800997A15 /* GoogleDriveRemoteFS.m in Sources */, + F8F2D9981986DCCC00997A15 /* S3GlacierRestorerParamSet.m in Sources */, F829521319868DE6001DC91B /* RegexKitLite.m in Sources */, F8F2D8FD1986B78600997A15 /* ArrayNode.m in Sources */, F8F2D9741986D02D00997A15 /* S3RestorerParamSet.m in Sources */, @@ -1320,6 +1390,7 @@ F8F2D9251986B98600997A15 /* BlobKey.m in Sources */, F829523319868E59001DC91B /* SBJsonParser.m in Sources */, F8F2D8A51986B67500997A15 /* Vault.m in Sources */, + F8F2D9A61986DE3B00997A15 /* GlacierPack.m in Sources */, F8F2D97F1986D3A100997A15 /* FileAttributes.m in Sources */, F829DC15198691CB00D637E0 /* NSErrorIO.m in Sources */, F82951EE19868D90001DC91B /* HTTPInputStream.m in Sources */, @@ -1381,6 +1452,7 @@ F8F2D8941986B66000997A15 /* GlacierRequest.m in Sources */, F829DBFF1986901300D637E0 /* Streams.m in Sources */, F8F2D8A41986B67500997A15 /* SHA256TreeHash.m in Sources */, + F8F2D9A81986DE3B00997A15 /* GlacierPackSet.m in Sources */, F8F2D9071986B78600997A15 /* XMLPListWriter.m in Sources */, F829DC1C1986921E00D637E0 /* MD5Hash.m in Sources */, F8F2D8681986B58000997A15 /* BackupSet.m in Sources */, @@ -1408,9 +1480,11 @@ F8F2D8AD1986B6A300997A15 /* SHA256Hash.m in Sources */, F8295168198683D5001DC91B /* session.c in Sources */, F8F2D86D1986B5CC00997A15 /* S3TargetConnection.m in Sources */, + F8F2D9AB1986DE4400997A15 /* GlacierRestorer.m in Sources */, F83F9B2D1983303F007CBFB4 /* ArqRestoreCommand.m in Sources */, F8F2D9341986BA2B00997A15 /* ArqSalt.m in Sources */, F829DC14198691CB00D637E0 /* IntegerIO.m in Sources */, + F8F2D9AE1986DE8300997A15 /* BinarySHA1.m in Sources */, F8F2D8FF1986B78600997A15 /* BinaryPListWriter.m in Sources */, F8F2D95B1986BE4E00997A15 /* PackIndex.m in Sources */, F829522319868E26001DC91B /* GoogleDriveRequest.m in Sources */, diff --git a/commonrestore/GlacierRequestItem.h b/commonrestore/GlacierRequestItem.h new file mode 100644 index 0000000..6096913 --- /dev/null +++ b/commonrestore/GlacierRequestItem.h @@ -0,0 +1,26 @@ +// +// GlacierRequestItem.h +// Arq +// +// Created by Stefan Reitshamer on 5/30/13. +// Copyright (c) 2013 Stefan Reitshamer. All rights reserved. +// + +@class Tree; +@class Node; +@protocol Restorer; +@class Repo; + + +@interface GlacierRequestItem : NSObject { + Tree *tree; + Node *node; + NSString *path; + BOOL requestedFirstBlobKey; + NSUInteger dataBlobKeyIndex; +} +- (id)initWithPath:(NSString *)thePath tree:(Tree *)theTree; +- (id)initWithPath:(NSString *)thePath node:(Node *)theNode; + +- (NSArray *)requestWithRestorer:(id )theRestorer repo:(Repo *)theRepo error:(NSError **)error; +@end diff --git a/commonrestore/GlacierRequestItem.m b/commonrestore/GlacierRequestItem.m new file mode 100644 index 0000000..89741ae --- /dev/null +++ b/commonrestore/GlacierRequestItem.m @@ -0,0 +1,106 @@ +// +// GlacierRequestItem.m +// Arq +// +// Created by Stefan Reitshamer on 5/30/13. +// Copyright (c) 2013 Stefan Reitshamer. All rights reserved. +// + +#import "GlacierRequestItem.h" +#import "Tree.h" +#import "Node.h" +#import "GlacierRestorer.h" +#import "BlobKey.h" +#import "Repo.h" + + +@implementation GlacierRequestItem +- (id)initWithPath:(NSString *)thePath tree:(Tree *)theTree { + if (self = [super init]) { + path = [thePath retain]; + tree = [theTree retain]; + } + return self; +} +- (id)initWithPath:(NSString *)thePath node:(Node *)theNode { + if (self = [super init]) { + path = [thePath retain]; + node = [theNode retain]; + } + return self; +} +- (id)initWithPath:(NSString *)thePath node:(Node *)theNode dataBlobKeyIndex:(NSUInteger)theDataBlobKeyIndex { + if (self = [super init]) { + path = [thePath retain]; + node = [theNode retain]; + dataBlobKeyIndex = theDataBlobKeyIndex; + NSAssert(theDataBlobKeyIndex > 0, @"theDataBlobKeyIndex must be > 0"); + requestedFirstBlobKey = YES; + } + return self; +} +- (void)dealloc { + [path release]; + [tree release]; + [node release]; + [super dealloc]; +} + +- (NSArray *)requestWithRestorer:(id )theRestorer repo:(Repo *)theRepo error:(NSError **)error { + NSMutableArray *nextItems = [NSMutableArray array]; + + if (tree != nil) { + if (![theRestorer requestBlobKey:[tree xattrsBlobKey] error:error]) { + return nil; + } + if (![theRestorer requestBlobKey:[tree aclBlobKey] error:error]) { + return nil; + } + for (NSString *childNodeName in [tree childNodeNames]) { + Node *childNode = [tree childNodeWithName:childNodeName]; + NSString *childPath = [path stringByAppendingPathComponent:childNodeName]; + [nextItems addObject:[[[GlacierRequestItem alloc] initWithPath:childPath node:childNode] autorelease]]; + } + } else { + NSAssert(node != nil, @"node can't be nil if tree is nil"); + if ([node isTree]) { + Tree *childTree = [theRepo treeForBlobKey:[node treeBlobKey] error:error]; + if (childTree == nil) { + return nil; + } + [nextItems addObject:[[[GlacierRequestItem alloc] initWithPath:path tree:childTree] autorelease]]; + } else { + if (!requestedFirstBlobKey) { + if (![theRestorer requestBlobKey:[node xattrsBlobKey] error:error]) { + return nil; + } + if (![theRestorer requestBlobKey:[node aclBlobKey] error:error]) { + return nil; + } + if (![theRestorer shouldSkipFile:path]) { + if ([[node dataBlobKeys] count] > 0) { + BlobKey *firstKey = [[node dataBlobKeys] objectAtIndex:0]; + HSLogDetail(@"requesting first data blob of %ld for %@", (unsigned long)[[node dataBlobKeys] count], path); + if (![theRestorer requestBlobKey:firstKey error:error]) { + return nil; + } + if ([[node dataBlobKeys] count] > 1) { + [nextItems addObject:[[[GlacierRequestItem alloc] initWithPath:path node:node dataBlobKeyIndex:1] autorelease]]; + } + } + } + } else { + BlobKey *blobKey = [[node dataBlobKeys] objectAtIndex:dataBlobKeyIndex]; + HSLogDetail(@"requesting data blob %ld of %ld for %@", (unsigned long)(dataBlobKeyIndex + 1), (unsigned long)[[node dataBlobKeys] count], path); + if (![theRestorer requestBlobKey:blobKey error:error]) { + return nil; + } + if ([[node dataBlobKeys] count] > (dataBlobKeyIndex + 1)) { + [nextItems addObject:[[[GlacierRequestItem alloc] initWithPath:path node:node dataBlobKeyIndex:(dataBlobKeyIndex + 1)] autorelease]]; + } + } + } + } + return nextItems; +} +@end diff --git a/glacierrestore/GlacierPack.h b/glacierrestore/GlacierPack.h new file mode 100644 index 0000000..fdf9980 --- /dev/null +++ b/glacierrestore/GlacierPack.h @@ -0,0 +1,38 @@ +// +// GlacierPack.h +// +// Created by Stefan Reitshamer on 11/3/12. +// Copyright (c) 2012 Stefan Reitshamer. All rights reserved. +// + +@class Target; + + +@interface GlacierPack : NSObject { + NSString *s3BucketName; + NSString *computerUUID; + NSString *bucketUUID; + NSString *packSetName; + NSString *packSHA1; + NSString *archiveId; + NSString *localPath; + unsigned long long packSize; + uid_t uid; + gid_t gid; +} +- (id)initWithTarget:(Target *)theTarget + s3BucketName:(NSString *)theS3BucketName + computerUUID:(NSString *)theComputerUUID + bucketUUID:(NSString *)theBucketUUID + packSHA1:(NSString *)thePackSHA1 + archiveId:(NSString *)theArchiveId + packSize:(unsigned long long)thePackSize + targetUID:(uid_t)theTargetUID + targetGID:(gid_t)theTargetGID; + +- (NSString *)packSHA1; +- (NSString *)archiveId; +- (unsigned long long)packSize; +- (BOOL)cachePackDataToDisk:(NSData *)thePackData error:(NSError **)error; +- (NSData *)cachedDataForObjectAtOffset:(unsigned long long)offset error:(NSError **)error; +@end diff --git a/glacierrestore/GlacierPack.m b/glacierrestore/GlacierPack.m new file mode 100644 index 0000000..b0ec3ae --- /dev/null +++ b/glacierrestore/GlacierPack.m @@ -0,0 +1,121 @@ +// +// GlacierPack.m +// +// Created by Stefan Reitshamer on 11/3/12. +// Copyright (c) 2012 Stefan Reitshamer. All rights reserved. +// + +#import "GlacierPack.h" +#import "UserLibrary_Arq.h" +#import "NSFileManager_extra.h" +#import "FDInputStream.h" +#import "BufferedInputStream.h" +#import "StringIO.h" +#import "IntegerIO.h" +#import "Streams.h" +#import "Target.h" + + +@implementation GlacierPack +- (id)initWithTarget:(Target *)theTarget + s3BucketName:(NSString *)theS3BucketName + computerUUID:(NSString *)theComputerUUID + bucketUUID:(NSString *)theBucketUUID + packSHA1:(NSString *)thePackSHA1 + archiveId:(NSString *)theArchiveId + packSize:(unsigned long long)thePackSize + targetUID:(uid_t)theTargetUID + targetGID:(gid_t)theTargetGID { + if (self = [super init]) { + s3BucketName = [theS3BucketName retain]; + computerUUID = [theComputerUUID retain]; + bucketUUID = [theBucketUUID retain]; + packSetName = [[NSString alloc] initWithFormat:@"%@-glacierblobs", theBucketUUID]; + packSHA1 = [thePackSHA1 retain]; + archiveId = [theArchiveId retain]; + packSize = thePackSize; + uid = theTargetUID; + gid = theTargetGID; + localPath = [[NSString alloc] initWithFormat:@"%@/%@/%@/glacier_packsets/%@/%@/%@.pack", + [UserLibrary arqCachePath], [theTarget targetUUID], computerUUID, packSetName, [packSHA1 substringToIndex:2], [packSHA1 substringFromIndex:2]]; + } + return self; +} +- (void)dealloc { + [s3BucketName release]; + [computerUUID release]; + [bucketUUID release]; + [packSetName release]; + [packSHA1 release]; + [archiveId release]; + [localPath release]; + [super dealloc]; +} + +- (NSString *)packSHA1 { + return packSHA1; +} +- (NSString *)archiveId { + return archiveId; +} +- (unsigned long long)packSize { + return packSize; +} +- (BOOL)cachePackDataToDisk:(NSData *)thePackData error:(NSError **)error { + if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:localPath targetUID:uid targetGID:gid error:error]) { + return NO; + } + return [Streams writeData:thePackData atomicallyToFile:localPath targetUID:uid targetGID:gid bytesWritten:NULL error:error]; +} +- (NSData *)cachedDataForObjectAtOffset:(unsigned long long)offset error:(NSError **)error { + int fd = open([localPath fileSystemRepresentation], O_RDONLY); + if (fd == -1) { + int errnum = errno; + HSLogError(@"open(%@) error %d: %s", localPath, errnum, strerror(errnum)); + SETNSERROR(@"UnixErrorDomain", errnum, @"failed to open %@: %s", localPath, strerror(errnum)); + return nil; + } + NSData *ret = nil; + FDInputStream *fdis = [[FDInputStream alloc] initWithFD:fd label:localPath]; + BufferedInputStream *bis = [[BufferedInputStream alloc] initWithUnderlyingStream:fdis]; + do { + if (lseek(fd, offset, SEEK_SET) == -1) { + int errnum = errno; + HSLogError(@"lseek(%@, %qu) error %d: %s", localPath, offset, errnum, strerror(errnum)); + SETNSERROR(@"UnixErrorDomain", errnum, @"failed to seek to %qu in %@: %s", offset, localPath, strerror(errnum)); + break; + } + NSString *mimeType; + NSString *downloadName; + if (![StringIO read:&mimeType from:bis error:error] || ![StringIO read:&downloadName from:bis error:error]) { + break; + } + uint64_t dataLen = 0; + if (![IntegerIO readUInt64:&dataLen from:bis error:error]) { + break; + } + NSData *data = nil; + if (dataLen > 0) { + unsigned char *buf = (unsigned char *)malloc((size_t)dataLen); + if (![bis readExactly:(NSUInteger)dataLen into:buf error:error]) { + free(buf); + break; + } + data = [NSData dataWithBytesNoCopy:buf length:(NSUInteger)dataLen]; + } else { + data = [NSData data]; + } + ret = data; + } while (0); + close(fd); + [bis release]; + [fdis release]; + return ret; +} + + +#pragma mark NSObject +- (NSString *)description { + return [NSString stringWithFormat:@"", packSHA1]; +} +@end diff --git a/glacierrestore/GlacierPackIndex.h b/glacierrestore/GlacierPackIndex.h new file mode 100644 index 0000000..ed390aa --- /dev/null +++ b/glacierrestore/GlacierPackIndex.h @@ -0,0 +1,53 @@ +// +// GlacierPackIndex.h +// +// Created by Stefan Reitshamer on 11/3/12. +// Copyright (c) 2012 Stefan Reitshamer. All rights reserved. +// + +@class S3Service; +@class PackIndexEntry; +@class PackId; +@class Target; +@protocol TargetConnectionDelegate; + + +@interface GlacierPackIndex : NSObject { + S3Service *s3; + NSString *s3BucketName; + NSString *computerUUID; + PackId *packId; + NSString *s3Path; + NSString *localPath; + uid_t targetUID; + gid_t targetGID; + NSMutableArray *pies; + NSString *archiveId; + unsigned long long packSize; +} ++ (NSString *)s3PathWithS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID packId:(PackId *)thePackId; ++ (NSString *)localPathWithTarget:(Target *)theTarget computerUUID:(NSString *)theComputerUUID packId:(PackId *)thePackId; ++ (NSArray *)glacierPackIndexesForTarget:(Target *)theTarget + s3Service:(S3Service *)theS3 + s3BucketName:theS3BucketName + computerUUID:(NSString *)theComputerUUID + packSetName:(NSString *)thePackSetName + targetConnectionDelegate:(id )theTCD + targetUID:(uid_t)theTargetUID + targetGID:(gid_t)theTargetGID + error:(NSError **)error; + +- (id)initWithTarget:(Target *)theTarget + s3Service:(S3Service *)theS3 + s3BucketName:(NSString *)theS3BucketName + computerUUID:(NSString *)theComputerUUID + packId:(PackId *)thePackId + targetUID:(uid_t)theTargetUID + targetGID:(gid_t)theTargetGID; +- (BOOL)makeLocalWithTargetConnectionDelegate:(id )theTCD error:(NSError **)error; +- (NSArray *)allPackIndexEntriesWithTargetConnectionDelegate:(id )theTCD error:(NSError **)error; +- (PackIndexEntry *)entryForSHA1:(NSString *)sha1 error:(NSError **)error; +- (PackId *)packId; +- (NSString *)archiveId:(NSError **)error; +- (unsigned long long)packSize:(NSError **)error; +@end diff --git a/glacierrestore/GlacierPackIndex.m b/glacierrestore/GlacierPackIndex.m new file mode 100644 index 0000000..f1cc1f2 --- /dev/null +++ b/glacierrestore/GlacierPackIndex.m @@ -0,0 +1,405 @@ +// +// GlacierPackIndex.m +// +// Created by Stefan Reitshamer on 11/3/12. +// Copyright (c) 2012 Stefan Reitshamer. All rights reserved. +// + +#include +#include +#include +#import "GlacierPackIndex.h" +#import "S3Service.h" +#import "RegexKitLite.h" +#import "NSString_extra.h" +#import "BinarySHA1.h" +#import "PackIndexEntry.h" +#import "FileOutputStream.h" +#import "Streams.h" +#import "NSFileManager_extra.h" +#import "UserLibrary_Arq.h" +#import "NSError_extra.h" +#import "DataInputStream.h" +#import "FDInputStream.h" +#import "BufferedInputStream.h" +#import "StringIO.h" +#import "IntegerIO.h" +#import "PackId.h" +#import "Target.h" + + +typedef struct index_object { + uint64_t nbo_offset; + uint64_t nbo_datalength; + unsigned char sha1[20]; + unsigned char filler[4]; +} index_object; + +typedef struct pack_index { + uint32_t magic_number; + uint32_t nbo_version; + uint32_t nbo_fanout[256]; + index_object first_index_object; +} pack_index; + + +@implementation GlacierPackIndex ++ (NSString *)s3PathWithS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID packId:(PackId *)thePackId { + return [NSString stringWithFormat:@"/%@/%@/packsets/%@/%@.index", theS3BucketName, theComputerUUID, [thePackId packSetName], [thePackId packSHA1]]; +} ++ (NSString *)localPathWithTarget:(Target *)theTarget computerUUID:(NSString *)theComputerUUID packId:(PackId *)thePackId { + return [NSString stringWithFormat:@"%@/%@/%@/glacier_packsets/%@/%@/%@.index", [UserLibrary arqCachePath], [theTarget targetUUID], theComputerUUID, [thePackId packSetName], [[thePackId packSHA1] substringToIndex:2], [[thePackId packSHA1] substringFromIndex:2]]; +} ++ (NSArray *)glacierPackIndexesForTarget:(Target *)theTarget + s3Service:(S3Service *)theS3 + s3BucketName:theS3BucketName + computerUUID:(NSString *)theComputerUUID + packSetName:(NSString *)thePackSetName + targetConnectionDelegate:(id )theTCD + targetUID:(uid_t)theTargetUID + targetGID:(gid_t)theTargetGID + error:(NSError **)error { + NSMutableArray *diskPackIndexes = [NSMutableArray array]; + NSString *packSetsPrefix = [NSString stringWithFormat:@"/%@/%@/packsets/%@/", theS3BucketName, theComputerUUID, thePackSetName]; + NSArray *paths = [theS3 pathsWithPrefix:packSetsPrefix targetConnectionDelegate:theTCD error:error]; + if (paths == nil) { + return nil; + } + for (NSString *thePath in paths) { + NSRange sha1Range = [thePath rangeOfRegex:@"/(\\w+)\\.index$" capture:1]; + if (sha1Range.location != NSNotFound) { + NSString *thePackSHA1 = [thePath substringWithRange:sha1Range]; + PackId *packId = [[[PackId alloc] initWithPackSetName:thePackSetName packSHA1:thePackSHA1] autorelease]; + GlacierPackIndex *index = [[GlacierPackIndex alloc] initWithTarget:theTarget + s3Service:theS3 + s3BucketName:theS3BucketName + computerUUID:theComputerUUID + packId:packId + targetUID:theTargetUID + targetGID:theTargetGID]; + [diskPackIndexes addObject:index]; + [index release]; + } + } + return diskPackIndexes; +} + + +- (id)initWithTarget:(Target *)theTarget + s3Service:(S3Service *)theS3 + s3BucketName:(NSString *)theS3BucketName + computerUUID:(NSString *)theComputerUUID + packId:(PackId *)thePackId + targetUID:(uid_t)theTargetUID + targetGID:(gid_t)theTargetGID { + if (self = [super init]) { + s3 = [theS3 retain]; + s3BucketName = [theS3BucketName retain]; + computerUUID = [theComputerUUID retain]; + packId = [thePackId retain]; + s3Path = [[GlacierPackIndex s3PathWithS3BucketName:s3BucketName computerUUID:computerUUID packId:packId] retain]; + localPath = [[GlacierPackIndex localPathWithTarget:theTarget computerUUID:computerUUID packId:packId] retain]; + targetUID = theTargetUID; + targetGID = theTargetGID; + } + return self; +} +- (void)dealloc { + [s3 release]; + [s3BucketName release]; + [computerUUID release]; + [packId release]; + [s3Path release]; + [localPath release]; + [pies release]; + [archiveId release]; + [super dealloc]; +} +- (BOOL)makeLocalWithTargetConnectionDelegate:(id)theTCD error:(NSError **)error { + NSFileManager *fm = [NSFileManager defaultManager]; + BOOL ret = YES; + if (![fm fileExistsAtPath:localPath]) { + for (;;) { + HSLogDebug(@"packset %@: making pack index %@ local", packId, packId); + NSError *myError = nil; + NSData *data = [s3 dataAtPath:s3Path targetConnectionDelegate:theTCD error:&myError]; + if (data != nil) { + ret = [self savePackIndex:data error:error]; + break; + } + if (![myError isTransientError]) { + HSLogError(@"error getting S3 pack index %@: %@", s3Path, myError); + if (error != NULL) { + *error = myError; + } + ret = NO; + break; + } else { + HSLogWarn(@"network error making pack index %@ local (retrying): %@", s3Path, myError); + NSError *rmError = nil; + if ([[NSFileManager defaultManager] fileExistsAtPath:localPath] && ![[NSFileManager defaultManager] removeItemAtPath:localPath error:&rmError]) { + HSLogError(@"error deleting incomplete downloaded pack index %@: %@", localPath, rmError); + } + } + } + } + return ret; +} +- (NSArray *)allPackIndexEntriesWithTargetConnectionDelegate:(id)theTCD error:(NSError **)error { + if (![self makeLocalWithTargetConnectionDelegate:theTCD error:error]) { + return nil; + } + if (![self readFile:error]) { + return nil; + } + return pies; +} + +- (PackIndexEntry *)entryForSHA1:(NSString *)sha1 error:(NSError **)error { + if (error != NULL) { + *error = nil; + } + NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; + PackIndexEntry *ret = [self doEntryForSHA1:sha1 error:(NSError **)error]; + [ret retain]; + if (ret == nil && error != NULL) { + [*error retain]; + } + [pool drain]; + [ret autorelease]; + if (ret == nil && error != NULL) { + [*error autorelease]; + } + return ret; +} +- (PackId *)packId { + return packId; +} +- (NSString *)archiveId:(NSError **)error { + if (![self readFile:error]) { + return nil; + } + return archiveId; +} +- (unsigned long long)packSize:(NSError **)error { + if (![self readFile:error]) { + return 0; + } + return packSize; +} + + +#pragma mark NSObject +- (NSString *)description { + return [NSString stringWithFormat:@"", computerUUID, packId]; +} + + +#pragma mark internal +- (BOOL)readFile:(NSError **)error { + if (pies != nil) { + return YES; + } + + int fd = open([localPath fileSystemRepresentation], O_RDONLY); + if (fd == -1) { + int errnum = errno; + HSLogError(@"open(%@) error %d: %s", localPath, errnum, strerror(errnum)); + SETNSERROR(@"UnixErrorDomain", errnum, @"failed to open %@: %s", localPath, strerror(errnum)); + return NO; + } + struct stat st; + if (fstat(fd, &st) == -1) { + int errnum = errno; + HSLogError(@"fstat(%@) error %d: %s", localPath, errnum, strerror(errnum)); + SETNSERROR(@"UnixErrorDomain", errnum, @"%@: %s", localPath, strerror(errnum)); + close(fd); + return NO; + } + if (st.st_size < sizeof(pack_index)) { + HSLogError(@"pack index length %ld is less than size of pack_index", (unsigned long)st.st_size); + SETNSERROR(@"GlacierPackIndexErrorDomain", -1, @"pack index length is less than size of pack_index"); + close(fd); + return NO; + } + pack_index *the_pack_index = mmap(0, (size_t)st.st_size, PROT_READ, MAP_SHARED, fd, 0); + if (the_pack_index == MAP_FAILED) { + int errnum = errno; + HSLogError(@"mmap(%@) error %d: %s", localPath, errnum, strerror(errnum)); + SETNSERROR(@"UnixErrorDomain", errnum, @"error mapping %@ to memory: %s", localPath, strerror(errnum)); + close(fd); + return NO; + } + pies = [[NSMutableArray alloc] init]; + uint32_t count = OSSwapBigToHostInt32(the_pack_index->nbo_fanout[255]); + index_object *indexObjects = &(the_pack_index->first_index_object); + for (uint32_t i = 0; i < count; i++) { + NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; + uint64_t offset = OSSwapBigToHostInt64(indexObjects[i].nbo_offset); + uint64_t dataLength = OSSwapBigToHostInt64(indexObjects[i].nbo_datalength); + NSString *objectSHA1 = [NSString hexStringWithBytes:indexObjects[i].sha1 length:20]; + PackIndexEntry *pie = [[[PackIndexEntry alloc] initWithPackId:packId offset:offset dataLength:dataLength objectSHA1:objectSHA1] autorelease]; + [pies addObject:pie]; + [pool drain]; + } + if (munmap(the_pack_index, (size_t)st.st_size) == -1) { + int errnum = errno; + HSLogError(@"munmap: %s", strerror(errnum)); + } + + uint32_t offset = sizeof(pack_index) + (count - 1) * sizeof(index_object); + if (!lseek(fd, offset, SEEK_SET) == -1) { + int errnum = errno; + HSLogError(@"lstat(%@, %ld) error %d: %s", localPath, (unsigned long)offset, errnum, strerror(errnum)); + SETNSERROR(@"UnixErrorDomain", errnum, @"error seeking to archiveId in pack index file"); + close(fd); + return NO; + } + FDInputStream *fdis = [[[FDInputStream alloc] initWithFD:fd label:@"packindex"] autorelease]; + BufferedInputStream *bis = [[[BufferedInputStream alloc] initWithUnderlyingStream:fdis] autorelease]; + BOOL ret = [StringIO read:&archiveId from:bis error:error] && [IntegerIO readUInt64:&packSize from:bis error:error]; + [archiveId retain]; + close(fd); + if (!ret) { + return NO; + } + return YES; +} +- (BOOL)savePackIndex:(NSData *)theData error:(NSError **)error { + if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:localPath targetUID:targetUID targetGID:targetGID error:error]) { + return NO; + } + id is = [[[DataInputStream alloc] initWithData:theData description:[self description]] autorelease]; + NSError *myError = nil; + unsigned long long written = 0; + BOOL ret = [Streams transferFrom:is atomicallyToFile:localPath targetUID:targetUID targetGID:targetGID bytesWritten:&written error:&myError]; + if (ret) { + HSLogDebug(@"wrote %qu bytes to %@", written, localPath); + } else { + if (error != NULL) { + *error = myError; + } + HSLogError(@"error making pack %@ local at %@: %@", packId, localPath, [myError localizedDescription]); + } + return ret; +} +- (PackIndexEntry *)doEntryForSHA1:(NSString *)sha1 error:(NSError **)error { + NSData *sha1Hex = [sha1 hexStringToData:error]; + if (sha1Hex == nil) { + return nil; + } + unsigned char *sha1Bytes = (unsigned char *)[sha1Hex bytes]; + HSLogTrace(@"looking for sha1 %@ in packindex %@", sha1, packId); + int fd = open([localPath fileSystemRepresentation], O_RDONLY); + if (fd == -1) { + int errnum = errno; + HSLogError(@"open(%@) error %d: %s", localPath, errnum, strerror(errnum)); + SETNSERROR(@"UnixErrorDomain", errnum, @"failed to open %@: %s", localPath, strerror(errnum)); + return nil; + } + uint32_t startIndex; + uint32_t endIndex; + if (![self readFanoutStartIndex:&startIndex fanoutEndIndex:&endIndex fromFD:fd forSHA1FirstByte:(unsigned int)sha1Bytes[0] error:error]) { + close(fd); + return nil; + } + close(fd); + if (endIndex == 0) { + SETNSERROR(@"PacksErrorDomain", ERROR_NOT_FOUND, @"sha1 %@ not found in pack", sha1); + return NO; + } + fd = open([localPath fileSystemRepresentation], O_RDONLY); + if (fd == -1) { + int errnum = errno; + HSLogError(@"open(%@) error %d: %s", localPath, errnum, strerror(errnum)); + SETNSERROR(@"UnixErrorDomain", errnum, @"failed to open %@: %s", localPath, strerror(errnum)); + return nil; + } + PackIndexEntry *ret = [self findEntryForSHA1:sha1 fd:fd betweenStartIndex:startIndex andEndIndex:endIndex error:error]; + close(fd); + if (ret != nil) { + HSLogTrace(@"found sha1 %@ in packindex %@", sha1, packId); + } + return ret; +} +- (PackIndexEntry *)findEntryForSHA1:(NSString *)sha1 fd:(int)fd betweenStartIndex:(uint32_t)startIndex andEndIndex:(uint32_t)endIndex error:(NSError **)error { + NSData *sha1Data = [sha1 hexStringToData:error]; + if (sha1Data == nil) { + return nil; + } + const void *sha1Bytes = [sha1Data bytes]; + uint32_t lengthToMap = 4 + 4 + 256*4 + endIndex * sizeof(index_object); + pack_index *the_pack_index = mmap(0, lengthToMap, PROT_READ, MAP_SHARED, fd, 0); + if (the_pack_index == MAP_FAILED) { + int errnum = errno; + HSLogError(@"mmap(%@) error %d: %s", localPath, errnum, strerror(errnum)); + SETNSERROR(@"UnixErrorDomain", errnum, @"error mapping %@ to memory: %s", localPath, strerror(errnum)); + return NO; + } + int64_t left = startIndex; + int64_t right = endIndex - 1; + int64_t middle; + int64_t offset; + int64_t dataLength; + PackIndexEntry *pie = nil; + while (left <= right) { + middle = (left + right)/2; + index_object *firstIndexObject = &(the_pack_index->first_index_object); + index_object *middleIndexObject = &firstIndexObject[middle]; + void *middleSHA1 = middleIndexObject->sha1; + NSComparisonResult result = [BinarySHA1 compare:middleSHA1 to:sha1Bytes]; + switch (result) { + case NSOrderedAscending: + left = middle + 1; + break; + case NSOrderedDescending: + right = middle - 1; + break; + default: + offset = OSSwapBigToHostInt64(middleIndexObject->nbo_offset); + dataLength = OSSwapBigToHostInt64(middleIndexObject->nbo_datalength); + pie = [[[PackIndexEntry alloc] initWithPackId:packId offset:offset dataLength:dataLength objectSHA1:sha1] autorelease]; + } + if (pie != nil) { + break; + } + } + if (munmap(the_pack_index, lengthToMap) == -1) { + int errnum = errno; + HSLogError(@"munmap: %s", strerror(errnum)); + } + if (pie == nil) { + SETNSERROR(@"PackErrorDomain", ERROR_NOT_FOUND, @"sha1 %@ not found in pack %@", sha1, packId); + } + return pie; +} +- (BOOL)readFanoutStartIndex:(uint32_t *)start fanoutEndIndex:(uint32_t *)end fromFD:(int)fd forSHA1FirstByte:(unsigned int)firstByte error:(NSError **)error { + size_t len = 4 + 4 + 4*256; + uint32_t *map = mmap(0, len, PROT_READ, MAP_SHARED, fd, 0); + if (map == MAP_FAILED) { + int errnum = errno; + HSLogError(@"mmap(%@) error %d: %s", localPath, errnum, strerror(errnum)); + SETNSERROR(@"UnixErrorDomain", errnum, @"error mapping %@ to memory: %s", localPath, strerror(errnum)); + return NO; + } + BOOL ret = YES; + uint32_t magicNumber = OSSwapBigToHostInt32(map[0]); + uint32_t version = OSSwapBigToHostInt32(map[1]); + if (magicNumber != 0xff744f63 || version != 2) { + SETNSERROR(@"PackErrorDomain", -1, @"invalid pack index header"); + ret = NO; + } else { + uint32_t *fanoutTable = map + 2; + *start = 0; + if (firstByte > 0) { + *start = OSSwapBigToHostInt32(fanoutTable[firstByte - 1]); + } + *end = OSSwapBigToHostInt32(fanoutTable[firstByte]); + } + if (munmap(map, len) == -1) { + int errnum = errno; + HSLogError(@"munmap: %s", strerror(errnum)); + } + return ret; +} +@end diff --git a/glacierrestore/GlacierPackSet.h b/glacierrestore/GlacierPackSet.h new file mode 100644 index 0000000..0a00149 --- /dev/null +++ b/glacierrestore/GlacierPackSet.h @@ -0,0 +1,47 @@ +// +// GlacierPackSet.h +// +// Created by Stefan Reitshamer on 11/3/12. +// Copyright (c) 2012 Stefan Reitshamer. All rights reserved. +// + +@class S3Service; +@class GlacierService; +@class PackIndexEntry; +@class GlacierPackIndex; +@class Target; +@protocol TargetConnectionDelegate; + + +@interface GlacierPackSet : NSObject { + Target *target; + S3Service *s3; + GlacierService *glacier; + NSString *s3BucketName; + NSString *computerUUID; + NSString *packSetName; + uid_t targetUID; + uid_t targetGID; + BOOL loadedPIEs; + + NSMutableDictionary *glacierPackIndexesByPackSHA1; + NSMutableDictionary *packIndexEntriesByObjectSHA1; +} ++ (NSString *)errorDomain; ++ (unsigned long long)maxPackFileSizeMB; ++ (unsigned long long)maxPackItemSizeBytes; + +- (id)initWithTarget:(Target *)theTarget + s3:(S3Service *)theS3 + glacier:(GlacierService *)theGlacier + vaultName:(NSString *)theVaultName + s3BucketName:(NSString *)theS3BucketName + computerUUID:(NSString *)theComputerUUID + packSetName:(NSString *)thePackSetName + targetUID:(uid_t)theTargetUID + targetGID:(uid_t)theTargetGID; + +- (BOOL)containsBlob:(BOOL *)contains forSHA1:(NSString *)sha1 dataSize:(unsigned long long *)dataSize targetConnectionDelegate:(id )theTCD error:(NSError **)error; +- (GlacierPackIndex *)glacierPackIndexForObjectSHA1:(NSString *)theObjectSHA1 targetConnectionDelegate:(id )theTCD error:(NSError **)error; +- (PackIndexEntry *)packIndexEntryForObjectSHA1:(NSString *)theSHA1 targetConnectionDelegate:(id )theTCD error:(NSError **)error; +@end diff --git a/glacierrestore/GlacierPackSet.m b/glacierrestore/GlacierPackSet.m new file mode 100644 index 0000000..05aff86 --- /dev/null +++ b/glacierrestore/GlacierPackSet.m @@ -0,0 +1,134 @@ +// +// GlacierPackSet.m +// +// Created by Stefan Reitshamer on 11/3/12. +// Copyright (c) 2012 Stefan Reitshamer. All rights reserved. +// + +#import "GlacierPackSet.h" +#import "PackIndexEntry.h" +#import "GlacierPackIndex.h" +#import "PackId.h" + + +static unsigned long long DEFAULT_MAX_PACK_FILE_SIZE_MB = 5; +static unsigned long long DEFAULT_MAX_PACK_ITEM_SIZE_BYTES = 65536; + + +@implementation GlacierPackSet ++ (NSString *)errorDomain { + return @"GlacierPackSetErrorDomain"; +} ++ (unsigned long long)maxPackFileSizeMB { + return DEFAULT_MAX_PACK_FILE_SIZE_MB; +} ++ (unsigned long long)maxPackItemSizeBytes { + return DEFAULT_MAX_PACK_ITEM_SIZE_BYTES; +} + +- (id)initWithTarget:(Target *)theTarget + s3:(S3Service *)theS3 + glacier:(GlacierService *)theGlacier + vaultName:(NSString *)theVaultName + s3BucketName:(NSString *)theS3BucketName + computerUUID:(NSString *)theComputerUUID + packSetName:(NSString *)thePackSetName + targetUID:(uid_t)theTargetUID + targetGID:(uid_t)theTargetGID { + if (self = [super init]) { + target = [theTarget retain]; + s3 = [theS3 retain]; + glacier = [theGlacier retain]; + s3BucketName = [theS3BucketName retain]; + computerUUID = [theComputerUUID retain]; + packSetName = [thePackSetName retain]; + targetUID = theTargetUID; + targetGID = theTargetGID; + glacierPackIndexesByPackSHA1 = [[NSMutableDictionary alloc] init]; + packIndexEntriesByObjectSHA1 = [[NSMutableDictionary alloc] init]; + } + return self; +} +- (void)dealloc { + [target release]; + [s3 release]; + [glacier release]; + [s3BucketName release]; + [computerUUID release]; + [packSetName release]; + [glacierPackIndexesByPackSHA1 release]; + [packIndexEntriesByObjectSHA1 release]; + [super dealloc]; +} + +- (BOOL)containsBlob:(BOOL *)contains forSHA1:(NSString *)sha1 dataSize:(unsigned long long *)dataSize targetConnectionDelegate:(id )theTCD error:(NSError **)error { + if (!loadedPIEs && ![self loadPackIndexEntriesWithTargetConnectionDelegate:theTCD error:error]) { + return NO; + } + PackIndexEntry *pie = [packIndexEntriesByObjectSHA1 objectForKey:sha1]; + *contains = (pie != nil); + if (pie != nil && dataSize != NULL) { + *dataSize = [pie dataLength]; + } + return YES; + +} +- (GlacierPackIndex *)glacierPackIndexForObjectSHA1:(NSString *)theObjectSHA1 targetConnectionDelegate:(id)theTCD error:(NSError **)error { + PackIndexEntry *pie = [self packIndexEntryForObjectSHA1:theObjectSHA1 targetConnectionDelegate:theTCD error:error]; + return [glacierPackIndexesByPackSHA1 objectForKey:[[pie packId] packSHA1]]; +} +- (PackIndexEntry *)packIndexEntryForObjectSHA1:(NSString *)theSHA1 targetConnectionDelegate:(id )theTCD error:(NSError **)error { + if (!loadedPIEs && ![self loadPackIndexEntriesWithTargetConnectionDelegate:theTCD error:error]) { + return NO; + } + PackIndexEntry *ret = [packIndexEntriesByObjectSHA1 objectForKey:theSHA1]; + if (ret == nil) { + SETNSERROR([GlacierPackSet errorDomain], ERROR_NOT_FOUND, @"object %@ not found in GlacierPackSet", theSHA1); + } + return ret; +} + + +#pragma mark internal +- (BOOL)loadPackIndexEntriesWithTargetConnectionDelegate:(id )theTCD error:(NSError **)error { + NSDictionary *thePackIndexEntriesByObjectSHA1 = [self packIndexEntriesBySHA1WithTargetConnectionDelegate:theTCD :error]; + if (thePackIndexEntriesByObjectSHA1 == nil) { + return NO; + } + [packIndexEntriesByObjectSHA1 setDictionary:thePackIndexEntriesByObjectSHA1]; + loadedPIEs = YES; + return YES; +} +- (NSDictionary *)packIndexEntriesBySHA1WithTargetConnectionDelegate:(id )theTCD :(NSError **)error { + NSArray *glacierPackIndexes = [GlacierPackIndex glacierPackIndexesForTarget:target s3Service:s3 s3BucketName:s3BucketName computerUUID:computerUUID packSetName:packSetName targetConnectionDelegate:theTCD targetUID:targetUID targetGID:targetGID error:error]; + if (glacierPackIndexes == nil) { + return nil; + } + NSMutableDictionary *packIndexEntriesBySHA1 = [NSMutableDictionary dictionary]; + for (GlacierPackIndex *index in glacierPackIndexes) { + if (![index makeLocalWithTargetConnectionDelegate:theTCD error:error]) { + return nil; + } + NSArray *pies = [index allPackIndexEntriesWithTargetConnectionDelegate:theTCD error:error]; + if (pies == nil) { + return nil; + } + unsigned long long packLength = 0; + for (PackIndexEntry *pie in pies) { + [packIndexEntriesBySHA1 setObject:pie forKey:[pie objectSHA1]]; + + unsigned long long endOffset = [pie offset] + [pie dataLength]; + if (endOffset > packLength) { + packLength = endOffset; + } + } + } + + [glacierPackIndexesByPackSHA1 removeAllObjects]; + for (GlacierPackIndex *index in glacierPackIndexes) { + [glacierPackIndexesByPackSHA1 setObject:index forKey:[[index packId] packSHA1]]; + } + + return packIndexEntriesBySHA1; +} +@end diff --git a/glacierrestore/GlacierRestorer.h b/glacierrestore/GlacierRestorer.h new file mode 100644 index 0000000..98de2fb --- /dev/null +++ b/glacierrestore/GlacierRestorer.h @@ -0,0 +1,69 @@ +// +// GlacierRestorer.h +// Arq +// +// Created by Stefan Reitshamer on 5/29/13. +// Copyright (c) 2013 Stefan Reitshamer. All rights reserved. +// + +#import "Restorer.h" +#import "TargetConnection.h" +@class GlacierRestorerParamSet; +@protocol GlacierRestorerDelegate; +@class SNS; +@class SQS; +@class S3Service; +@class GlacierService; +@class GlacierPackSet; +@class Repo; +@class Commit; +@class Tree; +@class BlobKey; + + +@interface GlacierRestorer : NSObject { + GlacierRestorerParamSet *paramSet; + id delegate; + + unsigned long long bytesToRequestPerRound; + NSDate *dateToResumeRequesting; + NSString *skipFilesRoot; + NSMutableDictionary *hardlinks; + NSString *jobUUID; + SNS *sns; + SQS *sqs; + S3Service *s3; + GlacierService *glacier; + GlacierPackSet *glacierPackSet; + NSMutableSet *requestedGlacierPackSHA1s; + NSMutableDictionary *requestedGlacierPacksByPackSHA1; + NSMutableArray *glacierPacksToDownload; + NSMutableArray *calculateItems; + NSMutableArray *glacierRequestItems; + NSMutableArray *restoreItems; + NSMutableSet *requestedArchiveIds; + + NSString *topicArn; + NSURL *queueURL; + NSString *queueArn; + NSString *subscriptionArn; + + Repo *repo; + Commit *commit; + NSString *commitDescription; + Tree *rootTree; + NSUInteger roundsCompleted; + unsigned long long bytesRequestedThisRound; + unsigned long long bytesRequested; + unsigned long long totalBytesToRequest; + + unsigned long long bytesTransferred; + unsigned long long totalBytesToTransfer; + + unsigned long long writtenToCurrentFile; +} +- (id)initWithGlacierRestorerParamSet:(GlacierRestorerParamSet *)theParamSet + delegate:(id )theDelegate; + +- (void)run; +@end diff --git a/glacierrestore/GlacierRestorer.m b/glacierrestore/GlacierRestorer.m new file mode 100644 index 0000000..7e6e016 --- /dev/null +++ b/glacierrestore/GlacierRestorer.m @@ -0,0 +1,964 @@ +// +// GlacierRestorer.m +// Arq +// +// Created by Stefan Reitshamer on 5/29/13. +// Copyright (c) 2013 Stefan Reitshamer. All rights reserved. +// + +#import "GlacierRestorer.h" +#import "GlacierRestorerParamSet.h" +#import "GlacierRestorerDelegate.h" +#import "S3AuthorizationProvider.h" +#import "S3Service.h" +#import "ArqSalt.h" +#import "Repo.h" +#import "Commit.h" +#import "Tree.h" +#import "Node.h" +#import "GlacierAuthorizationProvider.h" +#import "GlacierService.h" +#import "FileOutputStream.h" +#import "NSFileManager_extra.h" +#import "BlobKey.h" +#import "NSData-GZip.h" +#import "FileAttributes.h" +#import "BufferedOutputStream.h" +#import "OSStatusDescription.h" +#import "FileACL.h" +#import "BufferedInputStream.h" +#import "DataInputStream.h" +#import "XAttrSet.h" +#import "FileInputStream.h" +#import "SHA1Hash.h" +#import "PackIndexEntry.h" +#import "UserLibrary_Arq.h" +#import "SNS.h" +#import "SQS.h" +#import "NSString_extra.h" +#import "ReceiveMessageResponse.h" +#import "SQSMessage.h" +#import "NSObject+SBJSON.h" +#import "NSString+SBJSON.h" +#import "RestoreItem.h" +#import "GlacierRequestItem.h" +#import "CalculateItem.h" +#import "Bucket.h" +#import "Target.h" +#import "GlacierPackSet.h" +#import "GlacierPack.h" +#import "GlacierPackIndex.h" +#import "AWSRegion.h" +#import "Streams.h" + + +#define WAIT_TIME (3.0) +#define SLEEP_CYCLES (2) +#define MAX_QUEUE_MESSAGES_TO_READ (10) +#define MAX_GLACIER_RETRIES (10) + +#define RESTORE_DAYS (10) + + +@implementation GlacierRestorer +- (id)initWithGlacierRestorerParamSet:(GlacierRestorerParamSet *)theParamSet + delegate:(id )theDelegate { + if (self = [super init]) { + paramSet = [theParamSet retain]; + delegate = theDelegate; // Don't retain it. + + bytesToRequestPerRound = paramSet.downloadBytesPerSecond * 60 * 60 * 4; // 4 hours at preferred download rate + dateToResumeRequesting = [[NSDate date] retain]; + skipFilesRoot = [[[UserLibrary arqUserLibraryPath] stringByAppendingFormat:@"/RestoreJobSkipFiles/%f", [NSDate timeIntervalSinceReferenceDate]] retain]; + hardlinks = [[NSMutableDictionary alloc] init]; + jobUUID = [[NSString stringWithRandomUUID] retain]; + requestedGlacierPacksByPackSHA1 = [[NSMutableDictionary alloc] init]; + glacierPacksToDownload = [[NSMutableArray alloc] init]; + + calculateItems = [[NSMutableArray alloc] init]; + glacierRequestItems = [[NSMutableArray alloc] init]; + restoreItems = [[NSMutableArray alloc] init]; + requestedArchiveIds = [[NSMutableSet alloc] init]; + } + return self; +} +- (void)dealloc { + [paramSet release]; + + [dateToResumeRequesting release]; + [skipFilesRoot release]; + [hardlinks release]; + [jobUUID release]; + [sns release]; + [sqs release]; + [s3 release]; + [glacier release]; + [glacierPackSet release]; + [requestedGlacierPacksByPackSHA1 release]; + [glacierPacksToDownload release]; + [calculateItems release]; + [glacierRequestItems release]; + [restoreItems release]; + [requestedArchiveIds release]; + + [topicArn release]; + [queueURL release]; + [queueArn release]; + [subscriptionArn release]; + + [repo release]; + [commit release]; + [commitDescription release]; + [rootTree release]; + + [super dealloc]; +} + + +- (void)run { + HSLogDebug(@"GlacierRestorer starting"); + NSError *myError = nil; + if (![self run:&myError]) { + HSLogDebug(@"[GlacierRestorer run:] failed; %@", myError); + [delegate glacierRestorerDidFail:myError]; + } else { + HSLogDebug(@"[GlacierRestorer run:] succeeded"); + [delegate glacierRestorerDidSucceed]; + } + + [self deleteTopic]; + [self deleteQueue]; + + NSError *removeError = nil; + if ([[NSFileManager defaultManager] fileExistsAtPath:skipFilesRoot] && ![[NSFileManager defaultManager] removeItemAtPath:skipFilesRoot error:&removeError]) { + HSLogError(@"failed to remove %@: %@", skipFilesRoot, removeError); + } + + HSLogDebug(@"GlacierRestorer finished"); +} +- (NSNumber *)isObjectAvailableForBlobKey:(BlobKey *)theBlobKey error:(NSError **)error { + if ([theBlobKey storageType] == StorageTypeS3) { + // In Repo.m doPutData (line 503) we were incorrectly creating a BlobKey with storageType hard-coded to StorageTypeS3 when it should have been StorageTypeS3Glacier. + // Since we're here because we're restoring from a StorageTypeGlacier folder, we'll assume the storageType should be StorageTypeS3Glacier instead of StorageTypeS3. + theBlobKey = [[[BlobKey alloc] initCopyOfBlobKey:theBlobKey withStorageType:StorageTypeS3Glacier] autorelease]; + } + + if ([theBlobKey storageType] == StorageTypeS3Glacier) { + return [repo isObjectDownloadableForBlobKey:theBlobKey error:error]; + } + + // Packed blobs have sha1, but not archiveId. + if ([theBlobKey archiveId] == nil) { + return [NSNumber numberWithBool:YES]; + } + NSError *myError = nil; + NSString *jobId = [self completedJobIdForArchiveId:[theBlobKey archiveId] error:&myError]; + if (jobId == nil) { + if ([myError code] == ERROR_GLACIER_OBJECT_NOT_AVAILABLE) { + return [NSNumber numberWithBool:NO]; + } + SETERRORFROMMYERROR; + return nil; + } + return [NSNumber numberWithBool:YES]; +} +- (NSNumber *)sizeOfBlob:(BlobKey *)theBlobKey error:(NSError **)error { + if ([theBlobKey storageType] == StorageTypeGlacier) { + return [NSNumber numberWithUnsignedLongLong:[theBlobKey archiveSize]]; + } + + unsigned long long dataSize = 0; + NSNumber *contains = [repo containsBlobForBlobKey:theBlobKey dataSize:&dataSize error:error]; + if (contains == nil) { + return NO; + } + if (![contains boolValue]) { + // We'll report this to the user as an error during the download phase. + HSLogError(@"repo does not contain %@!", theBlobKey); + } + return [NSNumber numberWithUnsignedLongLong:dataSize]; +} +- (BOOL)requestBlobKey:(BlobKey *)theBlobKey error:(NSError **)error { + if (theBlobKey == nil) { + return YES; + } + + if ([theBlobKey storageType] == StorageTypeS3) { + // In Repo.m doPutData (line 503) we were incorrectly creating a BlobKey with storageType hard-coded to StorageTypeS3 when it should have been StorageTypeS3Glacier. + // Since we're here because we're restoring from a StorageTypeGlacier folder, we'll assume the storageType should be StorageTypeS3Glacier instead of StorageTypeS3. + theBlobKey = [[[BlobKey alloc] initCopyOfBlobKey:theBlobKey withStorageType:StorageTypeS3Glacier] autorelease]; + } + + if ([theBlobKey storageType] == StorageTypeS3Glacier) { + unsigned long long dataSize = 0; + NSNumber *contains = [repo containsBlobForBlobKey:theBlobKey dataSize:&dataSize error:error]; + if (contains == nil) { + return NO; + } + + if (![contains boolValue]) { + // We'll report this to the user as an error during the download phase. + HSLogError(@"repo does not contain %@!", theBlobKey); + } else { + BOOL alreadyRestoredOrRestoring = NO; + if (![repo restoreObjectForBlobKey:theBlobKey forDays:RESTORE_DAYS alreadyRestoredOrRestoring:&alreadyRestoredOrRestoring error:error]) { + return NO; + } + if (![self addToBytesRequested:dataSize error:error]) { + return NO; + } + } + return YES; + } + + + if ([theBlobKey archiveId] != nil) { + if (![requestedArchiveIds containsObject:[theBlobKey archiveId]]) { + if (![glacier initiateRetrievalJobForVaultName:[[paramSet bucket] vaultName] + archiveId:[theBlobKey archiveId] + snsTopicArn:topicArn + error:error]) { + return NO; + } + [requestedArchiveIds addObject:[theBlobKey archiveId]]; + HSLogDebug(@"requested %@", theBlobKey); + } + if (![self addToBytesRequested:[theBlobKey archiveSize] error:error]) { + return NO; + } + } + return YES; +} + + +#pragma mark Restorer +- (NSString *)errorDomain { + return @"GlacierRestorerErrorDomain"; +} +- (NSData *)dataForBlobKey:(BlobKey *)theBlobKey error:(NSError **)error { + if ([theBlobKey storageType] == StorageTypeS3) { + // In Repo.m doPutData (line 503) we were incorrectly creating a BlobKey with storageType hard-coded to StorageTypeS3 when it should have been StorageTypeS3Glacier. + // Since we're here because we're restoring from a StorageTypeGlacier folder, we'll assume the storageType should be StorageTypeS3Glacier instead of StorageTypeS3. + theBlobKey = [[[BlobKey alloc] initCopyOfBlobKey:theBlobKey withStorageType:StorageTypeS3Glacier] autorelease]; + } + + if ([theBlobKey storageType] == StorageTypeS3Glacier) { + NSData *data = [repo dataForBlobKey:theBlobKey error:error]; + if (data == nil) { + return nil; + } + if (![self addToBytesTransferred:(unsigned long long)[data length] error:error]) { + return nil; + } + return data; + } + + NSData *ret = nil; + if ([theBlobKey archiveId] == nil) { + // Packed blob. + PackIndexEntry *pie = [glacierPackSet packIndexEntryForObjectSHA1:[theBlobKey sha1] targetConnectionDelegate:self error:error]; + if (pie == nil) { + return nil; + } + GlacierPack *glacierPack = [requestedGlacierPacksByPackSHA1 objectForKey:[[pie packId] packSHA1]]; + if (glacierPack == nil) { + SETNSERROR([self errorDomain], -1, @"no GlacierPack for packSHA1 %@", [[pie packId] packSHA1]); + return nil; + } + ret = [glacierPack cachedDataForObjectAtOffset:[pie offset] error:error]; + } else { + NSString *completedJobId = [self completedJobIdForArchiveId:[theBlobKey archiveId] error:error]; + if (completedJobId == nil) { + return nil; + } + ret = [glacier dataForVaultName:[[paramSet bucket] vaultName] jobId:completedJobId retries:MAX_GLACIER_RETRIES error:error]; + if (ret != nil) { + if (![self addToBytesTransferred:[ret length] error:error]) { + return nil; + } + } + } + if (ret != nil) { + ret = [repo decryptData:ret error:error]; + } + return ret; +} +- (BOOL)shouldSkipFile:(NSString *)thePath { + NSString *skipFilePath = [skipFilesRoot stringByAppendingString:thePath]; + return [[NSFileManager defaultManager] fileExistsAtPath:skipFilePath]; +} +- (BOOL)useTargetUIDAndGID { + return paramSet.useTargetUIDAndGID; +} +- (uid_t)targetUID { + return paramSet.targetUID; +} +- (gid_t)targetGID { + return paramSet.targetGID; +} + + +#pragma mark TargetConnectionDelegate +- (BOOL)targetConnectionShouldRetryOnTransientError:(NSError **)error { + return YES; +} + + +#pragma mark internal +- (BOOL)run:(NSError **)error { + if (![self setUp:error]) { + return NO; + } + + NSString *calculatingMessage = @"Calculating sizes"; + if ([[NSFileManager defaultManager] fileExistsAtPath:paramSet.destinationPath]) { + calculatingMessage = @"Comparing existing files to backup data"; + } + if ([delegate glacierRestorerMessageDidChange:calculatingMessage]) { + SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested"); + return NO; + } + if (![self calculateSizes:error]) { + return NO; + } + + + if ([delegate glacierRestorerMessageDidChange:[NSString stringWithFormat:@"Restoring %@ from %@ to %@", paramSet.rootItemName, commitDescription, paramSet.destinationPath]]) { + SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested"); + return NO; + } + if (![self findNeededGlacierPacks:error]) { + return NO; + } + + // Just request all the Glacier packs right away. It probably won't amount to more than 4 hours' worth of downloads. + for (GlacierPack *glacierPack in [requestedGlacierPacksByPackSHA1 allValues]) { + if (![glacier initiateRetrievalJobForVaultName:[[paramSet bucket] vaultName] + archiveId:[glacierPack archiveId] + snsTopicArn:topicArn + error:error]) { + return NO; + } + if (![self addToBytesRequested:[glacierPack packSize] error:error]) { + return NO; + } + } + + + BOOL restoredAnItem = NO; + BOOL ret = YES; + NSAutoreleasePool *pool = nil; + for (;;) { + [pool drain]; + pool = [[NSAutoreleasePool alloc] init]; + + // Reset counters if necessary. + if (bytesRequestedThisRound >= bytesToRequestPerRound) { + roundsCompleted++; + bytesRequestedThisRound = 0; + NSDate *nextResumeDate = [[dateToResumeRequesting dateByAddingTimeInterval:(60 * 60 * 4)] retain]; + [dateToResumeRequesting release]; + dateToResumeRequesting = nextResumeDate; + HSLogDebug(@"reset next request resume date to %@", nextResumeDate); + } + + // Make sure we've transferred all the bytes from all but the most recent round of requests. + double theMinimum = (roundsCompleted = 0) ? 0 : ((double)bytesToRequestPerRound * (double)(roundsCompleted - 1)) * .9; + unsigned long long minimumBytesToHaveTransferred = (unsigned long long)theMinimum; + if ((bytesRequestedThisRound < bytesToRequestPerRound) + && (bytesTransferred >= minimumBytesToHaveTransferred) + && ([[NSDate date] earlierDate:dateToResumeRequesting] == dateToResumeRequesting)) { + + // Request more Glacier items. + if (![self requestMoreGlacierItems:error]) { + ret = NO; + break; + } + } + + if (!restoredAnItem) { + // Read any available items from the queue. + HSLogDebug(@"reading queue"); + if (![self readQueue:error]) { + ret = NO; + break; + } + } + + if ([glacierPacksToDownload count] == 0 && [restoreItems count] == 0) { + HSLogDebug(@"finished requesting"); + if ([delegate glacierRestorerDidFinishRequesting]) { + SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested"); + ret = NO; + break; + } + break; + } + + + // Restore an item if possible. + + if ([glacierPacksToDownload count] > 0) { + GlacierPack *glacierPack = [glacierPacksToDownload objectAtIndex:0]; + NSError *myError = nil; + NSString *completedJobId = [self completedJobIdForArchiveId:[glacierPack archiveId] error:&myError]; + if (completedJobId == nil) { + if ([myError code] != ERROR_GLACIER_OBJECT_NOT_AVAILABLE) { + SETERRORFROMMYERROR; + ret = NO; + break; + } + HSLogDebug(@"%@ not available yet", glacierPack); + restoredAnItem = NO; + } else { + HSLogDebug(@"downloading %@", glacierPack); + NSData *packData = [glacier dataForVaultName:[[paramSet bucket] vaultName] jobId:completedJobId retries:MAX_GLACIER_RETRIES error:error]; + if (packData == nil) { + ret = NO; + break; + } + restoredAnItem = YES; + if (![self addToBytesTransferred:[packData length] error:error]) { + ret = NO; + break; + } + if (![glacierPack cachePackDataToDisk:packData error:error]) { + ret = NO; + break; + } + HSLogDebug(@"downloaded %@", glacierPack); + [glacierPacksToDownload removeObject:glacierPack]; + restoredAnItem = YES; + } + + } else { + NSError *restoreError = nil; + RestoreItem *restoreItem = [restoreItems objectAtIndex:0]; + restoredAnItem = YES; + HSLogDebug(@"attempting to restore %@", restoreItem); + if (![restoreItem restoreWithHardlinks:hardlinks restorer:self error:&restoreError]) { + if ([restoreError code] == ERROR_GLACIER_OBJECT_NOT_AVAILABLE) { + HSLogDebug(@"glacier object not available yet"); + restoredAnItem = NO; + } else if ([restoreError isErrorWithDomain:[self errorDomain] code:ERROR_ABORT_REQUESTED]) { + if (error != NULL) { + *error = restoreError; + } + ret = NO; + break; + } else { + [delegate glacierRestorerErrorMessage:[restoreError localizedDescription] didOccurForPath:[restoreItem path]]; + } + } + if (restoredAnItem) { + NSArray *nextItems = [restoreItem nextItemsWithRepo:repo error:error]; + if (nextItems == nil) { + ret = NO; + break; + } + [restoreItems removeObjectAtIndex:0]; + if ([nextItems count] > 0) { + [restoreItems insertObjects:nextItems atIndexes:[NSIndexSet indexSetWithIndexesInRange:NSMakeRange(0, [nextItems count])]]; + } + } + } + + if (!restoredAnItem) { + HSLogDebug(@"sleeping"); + for (NSUInteger i = 0; i < SLEEP_CYCLES; i++) { + if (![self addToBytesTransferred:0 error:error]) { + ret = NO; + break; + } + [NSThread sleepForTimeInterval:3.0]; + } + } + if (!ret) { + break; + } + } + if (!ret && error != NULL) { + [*error retain]; + } + [pool drain]; + if (!ret && error != NULL) { + [*error autorelease]; + } + return ret; +} +- (BOOL)setUp:(NSError **)error { + NSString *secretAccessKey = [[[paramSet bucket] target] secret:error]; + if (secretAccessKey == nil) { + return NO; + } + + AWSRegion *awsRegion = [AWSRegion regionWithS3Endpoint:[[[paramSet bucket] target] endpoint]]; + if (awsRegion == nil) { + SETNSERROR([self errorDomain], -1, @"unknown AWS region %@", [[[paramSet bucket] target] endpoint]); + return NO; + } + sns = [[SNS alloc] initWithAccessKey:[[[[paramSet bucket] target] endpoint] user] secretKey:secretAccessKey awsRegion:awsRegion retryOnTransientError:YES]; + sqs = [[SQS alloc] initWithAccessKey:[[[[paramSet bucket] target] endpoint] user] secretKey:secretAccessKey awsRegion:awsRegion retryOnTransientError:YES]; + s3 = [[[[paramSet bucket] target] s3:error] retain]; + if (s3 == nil) { + return NO; + } + GlacierAuthorizationProvider *gap = [[[GlacierAuthorizationProvider alloc] initWithAccessKey:[[[[paramSet bucket] target] endpoint] user] secretKey:secretAccessKey] autorelease]; + glacier = [[GlacierService alloc] initWithGlacierAuthorizationProvider:gap awsRegion:awsRegion useSSL:YES retryOnTransientError:YES]; + glacierPackSet = [[GlacierPackSet alloc] initWithTarget:[[paramSet bucket] target] + s3:s3 + glacier:glacier + vaultName:[[paramSet bucket] vaultName] + s3BucketName:[[[[[paramSet bucket] target] endpoint] path] lastPathComponent] + computerUUID:[[paramSet bucket] computerUUID] + packSetName:[[[paramSet bucket] bucketUUID] stringByAppendingString:@"-glacierblobs"] + targetUID:paramSet.targetUID + targetGID:paramSet.targetGID]; + ArqSalt *arqSalt = [[[ArqSalt alloc] initWithTarget:[[paramSet bucket] target] targetUID:[paramSet targetUID] targetGID:[paramSet targetGID] computerUUID:[[paramSet bucket] computerUUID]] autorelease]; + NSData *salt = [arqSalt saltWithTargetConnectionDelegate:self error:error]; + if (salt == nil) { + return NO; + } + repo = [[Repo alloc] initWithBucket:[paramSet bucket] + encryptionPassword:[paramSet encryptionPassword] + targetUID:[paramSet targetUID] + targetGID:[paramSet targetGID] + loadExistingMutablePackFiles:NO + targetConnectionDelegate:self + repoDelegate:nil + error:error]; + if (repo == nil) { + return NO; + } + commit = [[repo commitForBlobKey:paramSet.commitBlobKey error:error] retain]; + if (commit == nil) { + return NO; + } + NSDateFormatter *dateFormatter = [[[NSDateFormatter alloc] init] autorelease]; + [dateFormatter setDateStyle:NSDateFormatterMediumStyle]; + [dateFormatter setTimeStyle:NSDateFormatterShortStyle]; + commitDescription = [[dateFormatter stringFromDate:[commit creationDate]] retain]; + + rootTree = [[repo treeForBlobKey:paramSet.treeBlobKey error:error] retain]; + if (rootTree == nil) { + return NO; + } + + if ([delegate glacierRestorerMessageDidChange:[NSString stringWithFormat:@"Creating SNS topic and SQS queue"]]) { + SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested"); + return NO; + } + + topicArn = [[sns createTopic:[NSString stringWithFormat:@"%@_topic", jobUUID] error:error] retain]; + if (topicArn == nil) { + return NO; + } + queueURL = [[sqs createQueueWithName:[NSString stringWithFormat:@"%@_queue", jobUUID] error:error] retain]; + if (queueURL == nil) { + return NO; + } + queueArn = [[sqs queueArnForQueueURL:queueURL error:error] retain]; + if (queueArn == nil) { + return NO; + } + if (![sqs setSendMessagePermissionToQueueURL:queueURL queueArn:queueArn forSourceArn:topicArn error:error]) { + return NO; + } + subscriptionArn = [[sns subscribeQueueArn:queueArn toTopicArn:topicArn error:error] retain]; + if (subscriptionArn == nil) { + return NO; + } + + if (paramSet.nodeName != nil) { + Node *node = [rootTree childNodeWithName:paramSet.nodeName]; + if ([[rootTree childNodeNames] isEqualToArray:[NSArray arrayWithObject:@"."]]) { + // The single-file case. + node = [rootTree childNodeWithName:@"."]; + } + NSAssert(node != nil, @"node can't be nil"); + + [calculateItems addObject:[[[CalculateItem alloc] initWithPath:paramSet.destinationPath node:node] autorelease]]; + [glacierRequestItems addObject:[[[GlacierRequestItem alloc] initWithPath:paramSet.destinationPath node:node] autorelease]]; + [restoreItems addObject:[[[RestoreItem alloc] initWithPath:paramSet.destinationPath tree:rootTree node:node] autorelease]]; + } else { + [calculateItems addObject:[[[CalculateItem alloc] initWithPath:paramSet.destinationPath tree:rootTree] autorelease]]; + [glacierRequestItems addObject:[[[GlacierRequestItem alloc] initWithPath:paramSet.destinationPath tree:rootTree] autorelease]]; + [restoreItems addObject:[[[RestoreItem alloc] initWithPath:paramSet.destinationPath tree:rootTree] autorelease]]; + } + + return YES; +} +- (BOOL)calculateSizes:(NSError **)error { + BOOL ret = YES; + NSAutoreleasePool *pool = nil; + while ([calculateItems count] > 0) { + [pool drain]; + pool = [[NSAutoreleasePool alloc] init]; + + CalculateItem *item = [calculateItems objectAtIndex:0]; + if (![item calculateWithRepo:repo restorer:self error:error]) { + ret = NO; + break; + } + for (NSString *path in [item filesToSkip]) { + [self skipFile:path]; + } + unsigned long long bytesToTransfer = [item bytesToTransfer]; + if (![self addToTotalBytesToRequest:bytesToTransfer error:error]) { + ret = NO; + break; + } + if (![self addToTotalBytesToTransfer:bytesToTransfer error:error]) { + ret = NO; + break; + } + [calculateItems removeObjectAtIndex:0]; + NSArray *nextItems = [item nextItems]; + if ([nextItems count] > 0) { + [calculateItems insertObjects:nextItems atIndexes:[NSIndexSet indexSetWithIndexesInRange:NSMakeRange(0, [nextItems count])]]; + } + } + if (!ret && error != NULL) { + [*error retain]; + } + [pool drain]; + if (!ret && error != NULL) { + [*error autorelease]; + } + + return ret; +} +- (void)skipFile:(NSString *)thePath { + NSString *skipFilePath = [skipFilesRoot stringByAppendingString:thePath]; + NSError *myError = nil; + if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:skipFilePath targetUID:paramSet.targetUID targetGID:paramSet.targetGID error:&myError]) { + HSLogError(@"error creating parent dir for %@: %@", skipFilePath, myError); + return; + } + if (![[NSFileManager defaultManager] touchFileAtPath:skipFilePath targetUID:paramSet.targetUID targetGID:paramSet.targetGID error:&myError]) { + HSLogError(@"error touching %@: %@", skipFilePath, myError); + } + HSLogDebug(@"skip file %@", thePath); +} + +- (BOOL)findNeededGlacierPacks:(NSError **)error { + HSLogDebug(@"finding needed glacier packs"); + if (paramSet.nodeName != nil) { + Node *node = [rootTree childNodeWithName:paramSet.nodeName]; + if ([[rootTree childNodeNames] isEqualToArray:[NSArray arrayWithObject:@"."]]) { + // The single-file case. + node = [rootTree childNodeWithName:@"."]; + } + NSAssert(node != nil, @"node can't be nil"); + if (![self findNeededGlacierPacksForNode:node path:paramSet.destinationPath error:error]) { + return NO; + } + } else { + if (![self findNeededGlacierPacksForTree:rootTree path:paramSet.destinationPath error:error]) { + return NO; + } + } + + [glacierPacksToDownload setArray:[requestedGlacierPacksByPackSHA1 allValues]]; + + HSLogDebug(@"found %ld needed glacier packs", (unsigned long)[requestedGlacierPacksByPackSHA1 count]); + return YES; +} +- (BOOL)findNeededGlacierPacksForTree:(Tree *)theTree path:(NSString *)thePath error:(NSError **)error { + HSLogDebug(@"requesting glacier packs for tree xattrs %@", [theTree xattrsBlobKey]); + if (![self findNeededGlacierPackForBlobKey:[theTree xattrsBlobKey] error:error]) { + return NO; + } + HSLogDebug(@"requesting glacier packs for tree acl %@", [theTree aclBlobKey]); + if (![self findNeededGlacierPackForBlobKey:[theTree aclBlobKey] error:error]) { + return NO; + } + + BOOL ret = YES; + NSAutoreleasePool *pool = nil; + for (NSString *nodeName in [theTree childNodeNames]) { + [pool drain]; + pool = [[NSAutoreleasePool alloc] init]; + Node *node = [theTree childNodeWithName:nodeName]; + NSString *childPath = [thePath stringByAppendingPathComponent:nodeName]; + if ([node isTree]) { + Tree *childTree = [repo treeForBlobKey:[node treeBlobKey] error:error]; + if (childTree == nil) { + ret = NO; + break; + } + if (![self findNeededGlacierPacksForTree:childTree path:childPath error:error]) { + ret = NO; + break; + } + } else { + if (![self findNeededGlacierPacksForNode:node path:childPath error:error]) { + ret = NO; + break; + } + } + } + if (!ret && error != NULL) { + [*error retain]; + } + [pool drain]; + if (!ret && error != NULL) { + [*error autorelease]; + } + return ret; +} +- (BOOL)findNeededGlacierPacksForNode:(Node *)theNode path:(NSString *)thePath error:(NSError **)error { + if (![self findNeededGlacierPackForBlobKey:[theNode xattrsBlobKey] error:error]) { + return NO; + } + if (![self findNeededGlacierPackForBlobKey:[theNode aclBlobKey] error:error]) { + return NO; + } + + BOOL ret = YES; + if (![self shouldSkipFile:thePath]) { + for (BlobKey *dataBlobKey in [theNode dataBlobKeys]) { + if (![self findNeededGlacierPackForBlobKey:dataBlobKey error:error]) { + ret = NO; + break; + } + } + } + return ret; +} +- (BOOL)findNeededGlacierPackForBlobKey:(BlobKey *)theBlobKey error:(NSError **)error { + // Packed blobs have sha1, but not archiveId. + if (theBlobKey != nil && [theBlobKey storageType] == StorageTypeGlacier && [theBlobKey archiveId] == nil) { + NSString *theSHA1 = [theBlobKey sha1]; + NSError *myError = nil; + GlacierPackIndex *glacierPackIndex = [glacierPackSet glacierPackIndexForObjectSHA1:theSHA1 targetConnectionDelegate:self error:&myError]; + if (glacierPackIndex == nil) { + if ([myError code] == ERROR_NOT_FOUND) { + HSLogError(@"object SHA1 %@ not found in any glacier pack index", theSHA1); + } else { + SETERRORFROMMYERROR; + return NO; + } + } else if (![[requestedGlacierPacksByPackSHA1 allKeys] containsObject:[[glacierPackIndex packId] packSHA1]]) { + NSString *archiveId = [glacierPackIndex archiveId:error]; + if (archiveId == nil) { + return NO; + } + unsigned long long packSize = [glacierPackIndex packSize:error]; + if (packSize == 0) { + return NO; + } + + HSLogDebug(@"need glacier pack SHA1 %@ archiveId %@", [[glacierPackIndex packId] packSHA1], archiveId); + + if (![self addToTotalBytesToRequest:packSize error:error]) { + return NO; + } + if (![self addToTotalBytesToTransfer:packSize error:error]) { + return NO; + } + + GlacierPack *glacierPack = [[[GlacierPack alloc] initWithTarget:[[paramSet bucket] target] + s3BucketName:[[[[[paramSet bucket] target] endpoint] path] lastPathComponent] + computerUUID:[[paramSet bucket] computerUUID] + bucketUUID:[[paramSet bucket] bucketUUID] + packSHA1:[[glacierPackIndex packId] packSHA1] + archiveId:archiveId + packSize:packSize + targetUID:[paramSet targetUID] + targetGID:[paramSet targetGID]] autorelease]; + [requestedGlacierPacksByPackSHA1 setObject:glacierPack forKey:[[glacierPackIndex packId] packSHA1]]; + } + } + return YES; +} + +- (BOOL)readQueue:(NSError **)error { + BOOL ret = YES; + NSAutoreleasePool *pool = nil; + for (;;) { + [pool drain]; + pool = [[NSAutoreleasePool alloc] init]; + ReceiveMessageResponse *response = [sqs receiveMessagesForQueueURL:queueURL maxMessages:MAX_QUEUE_MESSAGES_TO_READ error:error]; + if (response == nil) { + ret = NO; + break; + } + HSLogDebug(@"got %lu messages from queue", (unsigned long)[[response messages] count]); + if ([[response messages] count] == 0) { + break; + } + for (SQSMessage *msg in [response messages]) { + if (![self processMessage:msg error:error]) { + ret = NO; + break; + } + } + } + if (!ret && error != NULL) { + [*error retain]; + } + [pool drain]; + if (!ret && error != NULL) { + [*error autorelease]; + } + return ret; +} +- (BOOL)processMessage:(SQSMessage *)theMessage error:(NSError **)error { + NSDictionary *json = [[theMessage body] JSONValue:error]; + if (json == nil) { + return NO; + } + id msgJson = [json objectForKey:@"Message"]; + + // Sometimes it comes back as an NSString, sometimes already as an NSDictionary?! + if ([msgJson isKindOfClass:[NSString class]]) { + msgJson = [(NSString *)msgJson JSONValue:error]; + if (msgJson == nil) { + return NO; + } + } + NSDictionary *msgDict = (NSDictionary *)msgJson; + + NSString *archiveId = [msgDict objectForKey:@"ArchiveId"]; + NSString *jobId = [msgDict objectForKey:@"JobId"]; + NSNumber *completed = [msgDict objectForKey:@"Completed"]; + NSAssert([completed boolValue], @"Completed must be YES"); + + HSLogDetail(@"archiveId %@ is now available", archiveId); + + if (![self saveCompletedJobWithJobId:jobId archiveId:archiveId error:error]) { + return NO; + } + + NSError *myError = nil; + if (![sqs deleteMessageWithQueueURL:queueURL receiptHandle:[theMessage receiptHandle] error:&myError]) { + HSLogError(@"error deleting message %@ from queue %@: %@", [theMessage receiptHandle], queueURL, myError); + } + return YES; +} +- (BOOL)requestMoreGlacierItems:(NSError **)error { + BOOL ret = YES; + NSAutoreleasePool *pool = nil; + while (bytesRequestedThisRound < bytesToRequestPerRound && [glacierRequestItems count] > 0) { + [pool drain]; + pool = [[NSAutoreleasePool alloc] init]; + GlacierRequestItem *item = [glacierRequestItems objectAtIndex:0]; + NSArray *nextItems = [item requestWithRestorer:self repo:repo error:error]; + if (nextItems == nil) { + ret = NO; + break; + } + [glacierRequestItems removeObjectAtIndex:0]; + if ([nextItems count] > 0) { + [glacierRequestItems insertObjects:nextItems atIndexes:[NSIndexSet indexSetWithIndexesInRange:NSMakeRange(0, [nextItems count])]]; + } + } + if (!ret && error != NULL) { + [*error retain]; + } + [pool drain]; + if (!ret && error != NULL) { + [*error autorelease]; + } + return ret; +} + +- (NSString *)completedJobIdForArchiveId:(NSString *)theArchiveId error:(NSError **)error { + NSString *ret = nil; + NSString *statusPath = [self statusPathForArchiveId:theArchiveId]; + if ([[NSFileManager defaultManager] fileExistsAtPath:statusPath]) { + NSDictionary *attribs = [[NSFileManager defaultManager] attributesOfItemAtPath:statusPath error:error]; + if (attribs == nil) { + return NO; + } + if ([[attribs objectForKey:NSFileSize] unsignedLongLongValue] > 0) { + NSData *jobIdData = [NSData dataWithContentsOfFile:statusPath options:NSUncachedRead error:error]; + if (jobIdData == nil) { + return NO; + } + ret = [[[NSString alloc] initWithData:jobIdData encoding:NSUTF8StringEncoding] autorelease]; + } + } + if (!ret) { + SETNSERROR([self errorDomain], ERROR_GLACIER_OBJECT_NOT_AVAILABLE, @"object not available for archive %@", theArchiveId); + } + return ret; +} +- (BOOL)didRequestArchiveId:(NSString *)theArchiveId error:(NSError **)error { + NSString *path = [self statusPathForArchiveId:theArchiveId]; + NSData *emptyData = [NSData data]; + if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:path targetUID:paramSet.targetUID targetGID:paramSet.targetGID error:error] + || ![Streams writeData:emptyData atomicallyToFile:path targetUID:[paramSet targetUID] targetGID:[paramSet targetGID] bytesWritten:NULL error:error]) { + return NO; + } + return YES; +} +- (BOOL)saveCompletedJobWithJobId:(NSString *)theJobId archiveId:(NSString *)theArchiveId error:(NSError **)error { + NSString *path = [self statusPathForArchiveId:theArchiveId]; + NSData *jobIdData = [theJobId dataUsingEncoding:NSUTF8StringEncoding]; + if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:path targetUID:paramSet.targetUID targetGID:paramSet.targetGID error:error] + || ![Streams writeData:jobIdData atomicallyToFile:path targetUID:paramSet.targetUID targetGID:paramSet.targetGID bytesWritten:NULL error:error]) { + return NO; + } + return YES; +} +- (NSString *)statusPathForArchiveId:(NSString *)theArchiveId { + return [[UserLibrary arqUserLibraryPath] stringByAppendingFormat:@"/RestoreJobData/%@/%@/%@/%@", + [[paramSet bucket] computerUUID], + jobUUID, + [theArchiveId substringToIndex:2], + [theArchiveId substringFromIndex:2]]; +} +- (BOOL)addToBytesRequested:(unsigned long long)length error:(NSError **)error { + bytesRequested += length; + bytesRequestedThisRound += length; + if ([delegate glacierRestorerBytesRequestedDidChange:[NSNumber numberWithUnsignedLongLong:bytesRequested]]) { + SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested"); + return NO; + } + return YES; +} +- (BOOL)addToTotalBytesToRequest:(unsigned long long)length error:(NSError **)error { + totalBytesToRequest += length; + if ([delegate glacierRestorerTotalBytesToRequestDidChange:[NSNumber numberWithUnsignedLongLong:totalBytesToRequest]]) { + SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested"); + return NO; + } + return YES; +} +- (BOOL)addToBytesTransferred:(unsigned long long)length error:(NSError **)error { + bytesTransferred += length; + if ([delegate glacierRestorerBytesTransferredDidChange:[NSNumber numberWithUnsignedLongLong:bytesTransferred]]) { + SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested"); + return NO; + } + return YES; +} +- (BOOL)addToTotalBytesToTransfer:(unsigned long long)length error:(NSError **)error { + totalBytesToTransfer += length; + if ([delegate glacierRestorerTotalBytesToTransferDidChange:[NSNumber numberWithUnsignedLongLong:totalBytesToTransfer]]) { + SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested"); + return NO; + } + return YES; +} +- (void)deleteQueue { + if (queueURL != nil) { + NSError *myError = nil; + HSLogDetail(@"delete SQS queue %@", queueURL); + if (![sqs deleteQueue:queueURL error:&myError]) { + HSLogError(@"error deleting queue %@: %@", queueURL, myError); + } + } +} +- (void)deleteTopic { + if (topicArn != nil) { + NSError *myError = nil; + HSLogDetail(@"deleting SNS topic %@", topicArn); + if (![sns deleteTopicWithArn:topicArn error:&myError]) { + HSLogError(@"error deleting SNS topic %@: %@", topicArn, myError); + } + } +} +@end diff --git a/glacierrestore/GlacierRestorerDelegate.h b/glacierrestore/GlacierRestorerDelegate.h new file mode 100644 index 0000000..216a1cd --- /dev/null +++ b/glacierrestore/GlacierRestorerDelegate.h @@ -0,0 +1,27 @@ +// +// GlacierRestorerDelegate.h +// Arq +// +// Created by Stefan Reitshamer on 5/29/13. +// Copyright (c) 2013 Stefan Reitshamer. All rights reserved. +// + + +@protocol GlacierRestorerDelegate + +// Methods return YES if cancel is requested. + +- (BOOL)glacierRestorerMessageDidChange:(NSString *)message; + +- (BOOL)glacierRestorerBytesRequestedDidChange:(NSNumber *)theRequested; +- (BOOL)glacierRestorerTotalBytesToRequestDidChange:(NSNumber *)theMaxRequested; +- (BOOL)glacierRestorerDidFinishRequesting; + +- (BOOL)glacierRestorerBytesTransferredDidChange:(NSNumber *)theTransferred; +- (BOOL)glacierRestorerTotalBytesToTransferDidChange:(NSNumber *)theTotal; + +- (BOOL)glacierRestorerErrorMessage:(NSString *)theErrorMessage didOccurForPath:(NSString *)thePath; + +- (BOOL)glacierRestorerDidSucceed; +- (BOOL)glacierRestorerDidFail:(NSError *)error; +@end diff --git a/glacierrestore/GlacierRestorerParamSet.h b/glacierrestore/GlacierRestorerParamSet.h new file mode 100644 index 0000000..51a9aad --- /dev/null +++ b/glacierrestore/GlacierRestorerParamSet.h @@ -0,0 +1,58 @@ +// +// GlacierRestorerParamSet.h +// Arq +// +// Created by Stefan Reitshamer on 5/29/13. +// Copyright (c) 2013 Stefan Reitshamer. All rights reserved. +// + +@class AWSRegion; +@class BlobKey; +@class Bucket; + + +@interface GlacierRestorerParamSet : NSObject { + Bucket *bucket; + NSString *encryptionPassword; + double downloadBytesPerSecond; + BlobKey *commitBlobKey; + NSString *rootItemName; + int treeVersion; + BOOL treeIsCompressed; + BlobKey *treeBlobKey; + NSString *nodeName; + uid_t targetUID; + gid_t targetGID; + BOOL useTargetUIDAndGID; + NSString *destinationPath; + int logLevel; +} +- (id)initWithBucket:(Bucket *)theBucket + encryptionPassword:(NSString *)theEncryptionPassword +downloadBytesPerSecond:(double)theDownloadBytesPerSecond + commitBlobKey:(BlobKey *)theCommitBlobKey + rootItemName:(NSString *)theRootItemName + treeVersion:(int32_t)theTreeVersion + treeIsCompressed:(BOOL)theTreeIsCompressed + treeBlobKey:(BlobKey *)theTreeBlobKey + nodeName:(NSString *)theNodeName + targetUID:(uid_t)theTargetUID + targetGID:(gid_t)theTargetGID + useTargetUIDAndGID:(BOOL)theUseTargetUIDAndGID + destinationPath:(NSString *)theDestination + logLevel:(int)theLogLevel; + +@property (readonly, retain) Bucket *bucket; +@property (readonly, retain) NSString *encryptionPassword; +@property (readonly) double downloadBytesPerSecond; +@property (readonly, retain) BlobKey *commitBlobKey; +@property (readonly, retain) NSString *rootItemName; +@property (readonly, retain) BlobKey *treeBlobKey; +@property (readonly, retain) NSString *nodeName; +@property (readonly) uid_t targetUID; +@property (readonly) gid_t targetGID; +@property (readonly) BOOL useTargetUIDAndGID; +@property (readonly, retain) NSString *destinationPath; +@property (readonly) int logLevel; + +@end diff --git a/glacierrestore/GlacierRestorerParamSet.m b/glacierrestore/GlacierRestorerParamSet.m new file mode 100644 index 0000000..70a978c --- /dev/null +++ b/glacierrestore/GlacierRestorerParamSet.m @@ -0,0 +1,79 @@ +// +// GlacierRestorerParamSet.m +// Arq +// +// Created by Stefan Reitshamer on 5/29/13. +// Copyright (c) 2013 Stefan Reitshamer. All rights reserved. +// + +#import "GlacierRestorerParamSet.h" +#import "AWSRegion.h" +#import "BlobKey.h" +#import "StringIO.h" +#import "IntegerIO.h" +#import "BlobKeyIO.h" +#import "BooleanIO.h" +#import "DataIO.h" +#import "Tree.h" +#import "DoubleIO.h" +#import "Bucket.h" + + +@implementation GlacierRestorerParamSet +@synthesize bucket; +@synthesize encryptionPassword; +@synthesize downloadBytesPerSecond; +@synthesize commitBlobKey; +@synthesize rootItemName; +@synthesize treeBlobKey; +@synthesize nodeName; +@synthesize targetUID; +@synthesize targetGID; +@synthesize useTargetUIDAndGID; +@synthesize destinationPath; +@synthesize logLevel; + + +- (id)initWithBucket:(Bucket *)theBucket + encryptionPassword:(NSString *)theEncryptionPassword +downloadBytesPerSecond:(double)theDownloadBytesPerSecond + commitBlobKey:(BlobKey *)theCommitBlobKey + rootItemName:(NSString *)theRootItemName + treeVersion:(int32_t)theTreeVersion + treeIsCompressed:(BOOL)theTreeIsCompressed + treeBlobKey:(BlobKey *)theTreeBlobKey + nodeName:(NSString *)theNodeName + targetUID:(uid_t)theTargetUID + targetGID:(gid_t)theTargetGID + useTargetUIDAndGID:(BOOL)theUseTargetUIDAndGID + destinationPath:(NSString *)theDestination + logLevel:(int)theLogLevel { + if (self = [super init]) { + bucket = [theBucket retain]; + encryptionPassword = [theEncryptionPassword retain]; + downloadBytesPerSecond = theDownloadBytesPerSecond; + commitBlobKey = [theCommitBlobKey retain]; + rootItemName = [theRootItemName retain]; + treeVersion = theTreeVersion; + treeIsCompressed = theTreeIsCompressed; + treeBlobKey = [theTreeBlobKey retain]; + nodeName = [theNodeName retain]; + targetUID = theTargetUID; + targetGID = theTargetGID; + useTargetUIDAndGID = theUseTargetUIDAndGID; + destinationPath = [theDestination retain]; + logLevel = theLogLevel; + } + return self; +} +- (void)dealloc { + [bucket release]; + [encryptionPassword release]; + [commitBlobKey release]; + [rootItemName release]; + [treeBlobKey release]; + [nodeName release]; + [destinationPath release]; + [super dealloc]; +} +@end diff --git a/repo/BinarySHA1.h b/repo/BinarySHA1.h new file mode 100644 index 0000000..39150a1 --- /dev/null +++ b/repo/BinarySHA1.h @@ -0,0 +1,14 @@ +// +// BinarySHA1.h +// +// Created by Stefan Reitshamer on 12/30/09. +// Copyright 2009 Haystack Software. All rights reserved. +// + + + + +@interface BinarySHA1 : NSObject { +} ++ (NSComparisonResult)compare:(const void *)a to:(const void *)b; +@end diff --git a/repo/BinarySHA1.m b/repo/BinarySHA1.m new file mode 100644 index 0000000..110047f --- /dev/null +++ b/repo/BinarySHA1.m @@ -0,0 +1,25 @@ +// +// BinarySHA1.m +// +// Created by Stefan Reitshamer on 12/30/09. +// Copyright 2009 Haystack Software. All rights reserved. +// + +#import "BinarySHA1.h" + + +@implementation BinarySHA1 ++ (NSComparisonResult)compare:(const void *)a to:(const void *)b { + unsigned char *left = (unsigned char *)a; + unsigned char *right = (unsigned char *)b; + for (int i = 0; i < 20; i++) { + if (left[i] < right[i]) { + return NSOrderedAscending; + } + if (left[i] > right[i]) { + return NSOrderedDescending; + } + } + return NSOrderedSame; +} +@end diff --git a/s3glacierrestore/S3GlacierRestorer.h b/s3glacierrestore/S3GlacierRestorer.h new file mode 100644 index 0000000..0a9ba79 --- /dev/null +++ b/s3glacierrestore/S3GlacierRestorer.h @@ -0,0 +1,50 @@ +// +// S3GlacierRestorer.h +// Arq +// +// Created by Stefan Reitshamer on 1/9/14. +// Copyright (c) 2014 Stefan Reitshamer. All rights reserved. +// + +#import "TargetConnection.h" +#import "Restorer.h" +@class S3GlacierRestorerParamSet; +@protocol S3GlacierRestorerDelegate; +@class Repo; +@class Commit; +@class Tree; + + + +@interface S3GlacierRestorer : NSObject { + S3GlacierRestorerParamSet *paramSet; + id delegate; + + Repo *repo; + Commit *commit; + Tree *rootTree; + + NSMutableArray *calculateItems; + NSMutableArray *glacierRequestItems; + NSMutableArray *restoreItems; + + NSString *skipFilesRoot; + + unsigned long long bytesActuallyRequestedThisRound; + unsigned long long bytesRequested; + unsigned long long totalBytesToRequest; + + unsigned long long bytesTransferred; + unsigned long long totalBytesToTransfer; + + unsigned long long bytesToRequestPerRound; + NSDate *dateToResumeRequesting; + NSUInteger roundsCompleted; + NSMutableDictionary *hardlinks; + + NSUInteger sleepCycles; +} +- (id)initWithS3GlacierRestorerParamSet:(S3GlacierRestorerParamSet *)theParamSet delegate:(id )theDelegate; + +- (void)run; +@end diff --git a/s3glacierrestore/S3GlacierRestorer.m b/s3glacierrestore/S3GlacierRestorer.m new file mode 100644 index 0000000..2d2467b --- /dev/null +++ b/s3glacierrestore/S3GlacierRestorer.m @@ -0,0 +1,445 @@ +// +// S3GlacierRestorer.m +// Arq +// +// Created by Stefan Reitshamer on 1/9/14. +// Copyright (c) 2014 Stefan Reitshamer. All rights reserved. +// + +#import "S3GlacierRestorer.h" +#import "S3GlacierRestorerParamSet.h" +#import "S3GlacierRestorerDelegate.h" +#import "commit.h" +#import "Tree.h" +#import "Node.h" +#import "Repo.h" +#import "CalculateItem.h" +#import "GlacierRequestItem.h" +#import "RestoreItem.h" +#import "NSFileManager_extra.h" +#import "UserLibrary_Arq.h" +#import "BlobKey.h" +#import "Bucket.h" +#import "Target.h" +#import "S3Service.h" + + +#define RESTORE_DAYS (10) + +#define SLEEP_CYCLES_START (1) +#define SLEEP_CYCLES_MAX (30) +#define SLEEP_CYCLE_DURATION (2.0) + + +@implementation S3GlacierRestorer +- (id)initWithS3GlacierRestorerParamSet:(S3GlacierRestorerParamSet *)theParamSet delegate:(id )theDelegate { + if (self = [super init]) { + paramSet = [theParamSet retain]; + delegate = theDelegate; + + calculateItems = [[NSMutableArray alloc] init]; + glacierRequestItems = [[NSMutableArray alloc] init]; + restoreItems = [[NSMutableArray alloc] init]; + + skipFilesRoot = [[[UserLibrary arqUserLibraryPath] stringByAppendingFormat:@"/RestoreJobSkipFiles/%f", [NSDate timeIntervalSinceReferenceDate]] retain]; + + bytesToRequestPerRound = paramSet.downloadBytesPerSecond * 60 * 60 * 4; // 4 hours at preferred download rate + dateToResumeRequesting = [[NSDate date] retain]; + hardlinks = [[NSMutableDictionary alloc] init]; + + sleepCycles = SLEEP_CYCLES_START; + } + return self; +} +- (void)dealloc { + [paramSet release]; + [repo release]; + [commit release]; + [rootTree release]; + [calculateItems release]; + [glacierRequestItems release]; + [restoreItems release]; + [skipFilesRoot release]; + [dateToResumeRequesting release]; + [hardlinks release]; + [super dealloc]; +} + +- (NSString *)errorDomain { + return @"S3GlacierRestorerErrorDomain"; +} + +- (void)run { + NSError *myError = nil; + if (![self run:&myError]) { + [delegate s3GlacierRestorerDidFail:myError]; + } else { + [delegate s3GlacierRestorerDidSucceed]; + } + + + NSError *removeError = nil; + if ([[NSFileManager defaultManager] fileExistsAtPath:skipFilesRoot] && ![[NSFileManager defaultManager] removeItemAtPath:skipFilesRoot error:&removeError]) { + HSLogError(@"failed to remove %@: %@", skipFilesRoot, removeError); + } + + HSLogDebug(@"S3GlacierRestorer finished"); +} + + +#pragma mark Restorer +- (BOOL)requestBlobKey:(BlobKey *)theBlobKey error:(NSError **)error { + if (theBlobKey == nil) { + return YES; + } + unsigned long long dataSize = 0; + NSNumber *contains = [repo containsBlobForBlobKey:theBlobKey dataSize:&dataSize error:error]; + if (contains == nil) { + return NO; + } + + if (![contains boolValue]) { + // We'll report this to the user as an error during the download phase. + HSLogError(@"repo does not contain %@!", theBlobKey); + } else { + BOOL alreadyRestoredOrRestoring = NO; + if (![repo restoreObjectForBlobKey:theBlobKey forDays:RESTORE_DAYS alreadyRestoredOrRestoring:&alreadyRestoredOrRestoring error:error]) { + return NO; + } + unsigned long long actualBytesRequested = alreadyRestoredOrRestoring ? 0 : dataSize; + if (![self addToBytesRequested:dataSize actualBytesRequested:actualBytesRequested error:error]) { + return NO; + } + + } + return YES; +} +- (NSNumber *)isObjectAvailableForBlobKey:(BlobKey *)theBlobKey error:(NSError **)error { + return [repo isObjectDownloadableForBlobKey:theBlobKey error:error]; +} +- (NSNumber *)sizeOfBlob:(BlobKey *)theBlobKey error:(NSError **)error { + unsigned long long size = 0; + NSNumber *contains = [repo containsBlobForBlobKey:theBlobKey dataSize:&size error:error]; + if (contains == nil) { + return nil; + } + if (![contains boolValue]) { + SETNSERROR([self errorDomain], ERROR_NOT_FOUND, @"size of blob %@ not found because blob not found", theBlobKey); + return NO; + } + return [NSNumber numberWithUnsignedLongLong:size]; +} +- (NSData *)dataForBlobKey:(BlobKey *)theBlobKey error:(NSError **)error { + NSData *data = [repo dataForBlobKey:theBlobKey error:error]; + if (data == nil) { + return nil; + } + if (![self addToBytesTransferred:(unsigned long long)[data length] error:error]) { + return nil; + } + return data; +} +- (BOOL)shouldSkipFile:(NSString *)thePath { + NSString *skipFilePath = [skipFilesRoot stringByAppendingString:thePath]; + return [[NSFileManager defaultManager] fileExistsAtPath:skipFilePath]; +} +- (BOOL)useTargetUIDAndGID { + return paramSet.useTargetUIDAndGID; +} +- (uid_t)targetUID { + return paramSet.targetUID; +} +- (gid_t)targetGID { + return paramSet.targetGID; +} + + +#pragma mark internal +- (BOOL)run:(NSError **)error { + repo = [[Repo alloc] initWithBucket:[paramSet bucket] + encryptionPassword:[paramSet encryptionPassword] + targetUID:[paramSet targetUID] + targetGID:[paramSet targetGID] + loadExistingMutablePackFiles:NO + targetConnectionDelegate:self + repoDelegate:nil + error:error]; + if (repo == nil) { + return NO; + } + + commit = [[repo commitForBlobKey:[paramSet commitBlobKey] dataSize:NULL error:error] retain]; + if (commit == nil) { + return NO; + } + + rootTree = [[repo treeForBlobKey:[paramSet treeBlobKey] dataSize:NULL error:error] retain]; + if (rootTree == nil) { + return NO; + } + + if (paramSet.nodeName != nil) { + Node *node = [rootTree childNodeWithName:paramSet.nodeName]; + if ([[rootTree childNodeNames] isEqualToArray:[NSArray arrayWithObject:@"."]]) { + // The single-file case. + node = [rootTree childNodeWithName:@"."]; + } + NSAssert(node != nil, @"node can't be nil"); + + [calculateItems addObject:[[[CalculateItem alloc] initWithPath:paramSet.destinationPath node:node] autorelease]]; + [glacierRequestItems addObject:[[[GlacierRequestItem alloc] initWithPath:paramSet.destinationPath node:node] autorelease]]; + [restoreItems addObject:[[[RestoreItem alloc] initWithPath:paramSet.destinationPath tree:rootTree node:node] autorelease]]; + } else { + [calculateItems addObject:[[[CalculateItem alloc] initWithPath:paramSet.destinationPath tree:rootTree] autorelease]]; + [glacierRequestItems addObject:[[[GlacierRequestItem alloc] initWithPath:paramSet.destinationPath tree:rootTree] autorelease]]; + [restoreItems addObject:[[[RestoreItem alloc] initWithPath:paramSet.destinationPath tree:rootTree] autorelease]]; + } + + NSString *calculatingMessage = @"Calculating sizes"; + if ([[NSFileManager defaultManager] fileExistsAtPath:paramSet.destinationPath]) { + calculatingMessage = @"Comparing existing files to backup data"; + } + if ([delegate s3GlacierRestorerMessageDidChange:calculatingMessage]) { + SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested"); + return NO; + } + if (![self calculateSizes:error]) { + return NO; + } + + + NSDateFormatter *dateFormatter = [[[NSDateFormatter alloc] init] autorelease]; + [dateFormatter setDateStyle:NSDateFormatterMediumStyle]; + [dateFormatter setTimeStyle:NSDateFormatterShortStyle]; + NSString *commitDescription = [dateFormatter stringFromDate:[commit creationDate]]; + if ([delegate s3GlacierRestorerMessageDidChange:[NSString stringWithFormat:@"Restoring %@ from %@ to %@", paramSet.rootItemName, commitDescription, paramSet.destinationPath]]) { + SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested"); + return NO; + } + + + BOOL restoredAnItem = NO; + BOOL ret = YES; + NSAutoreleasePool *pool = nil; + while ([restoreItems count] > 0) { + [pool drain]; + pool = [[NSAutoreleasePool alloc] init]; + + if ([glacierRequestItems count] > 0) { + // Reset counters if necessary. + if (bytesActuallyRequestedThisRound >= bytesToRequestPerRound) { + roundsCompleted++; + bytesActuallyRequestedThisRound = 0; + NSDate *nextResumeDate = [[dateToResumeRequesting dateByAddingTimeInterval:(60 * 60 * 4)] retain]; + [dateToResumeRequesting release]; + dateToResumeRequesting = nextResumeDate; + HSLogDebug(@"reset next request resume date to %@", nextResumeDate); + } + + // Make sure we've transferred all the bytes from all but the most recent round of requests. + double theMinimum = (roundsCompleted = 0) ? 0 : ((double)bytesToRequestPerRound * (double)(roundsCompleted - 1)) * .9; + unsigned long long minimumBytesToHaveTransferred = (unsigned long long)theMinimum; + if ((bytesActuallyRequestedThisRound < bytesToRequestPerRound) + && (bytesTransferred >= minimumBytesToHaveTransferred) + && ([[NSDate date] earlierDate:dateToResumeRequesting] == dateToResumeRequesting)) { + + // Request more Glacier items. + if (![self requestMoreGlacierItems:error]) { + ret = NO; + break; + } + } + + if ([glacierRequestItems count] == 0) { + HSLogDebug(@"finished requesting"); + if ([delegate s3GlacierRestorerDidFinishRequesting]) { + SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested"); + ret = NO; + break; + } + } + } + + + // Restore an item if possible. + + NSError *restoreError = nil; + RestoreItem *restoreItem = [restoreItems objectAtIndex:0]; + restoredAnItem = YES; + HSLogDebug(@"attempting to restore %@", restoreItem); + if (![restoreItem restoreWithHardlinks:hardlinks restorer:self error:&restoreError]) { + if ([restoreError isErrorWithDomain:[restoreItem errorDomain] code:ERROR_GLACIER_OBJECT_NOT_AVAILABLE]) { + HSLogDebug(@"glacier object not available yet"); + restoredAnItem = NO; + } else if ([restoreError isErrorWithDomain:[self errorDomain] code:ERROR_ABORT_REQUESTED]) { + if (error != NULL) { + *error = restoreError; + } + ret = NO; + break; + } else { + [delegate s3GlacierRestorerErrorMessage:[restoreError localizedDescription] didOccurForPath:[restoreItem path]]; + } + } + if (restoredAnItem) { + NSArray *nextItems = [restoreItem nextItemsWithRepo:repo error:error]; + if (nextItems == nil) { + ret = NO; + break; + } + [restoreItems removeObjectAtIndex:0]; + if ([nextItems count] > 0) { + [restoreItems insertObjects:nextItems atIndexes:[NSIndexSet indexSetWithIndexesInRange:NSMakeRange(0, [nextItems count])]]; + } + } + + if (!restoredAnItem) { + HSLogDebug(@"sleeping"); + + for (NSUInteger i = 0; i < sleepCycles; i++) { + if (![self addToBytesTransferred:0 error:error]) { + ret = NO; + break; + } + [NSThread sleepForTimeInterval:SLEEP_CYCLE_DURATION]; + } + sleepCycles *= 2; + if (sleepCycles > SLEEP_CYCLES_MAX) { + sleepCycles = SLEEP_CYCLES_MAX; + } + } + if (!ret) { + break; + } + } + if (!ret && error != NULL) { + [*error retain]; + } + [pool drain]; + if (!ret && error != NULL) { + [*error autorelease]; + } + return ret; +} + +- (BOOL)calculateSizes:(NSError **)error { + BOOL ret = YES; + NSAutoreleasePool *pool = nil; + while ([calculateItems count] > 0) { + [pool drain]; + pool = [[NSAutoreleasePool alloc] init]; + + CalculateItem *item = [calculateItems objectAtIndex:0]; + if (![item calculateWithRepo:repo restorer:self error:error]) { + ret = NO; + break; + } + for (NSString *path in [item filesToSkip]) { + [self skipFile:path]; + } + unsigned long long bytesToTransfer = [item bytesToTransfer]; + if (![self addToTotalBytesToRequest:bytesToTransfer error:error]) { + ret = NO; + break; + } + if (![self addToTotalBytesToTransfer:bytesToTransfer error:error]) { + ret = NO; + break; + } + [calculateItems removeObjectAtIndex:0]; + NSArray *nextItems = [item nextItems]; + if ([nextItems count] > 0) { + [calculateItems insertObjects:nextItems atIndexes:[NSIndexSet indexSetWithIndexesInRange:NSMakeRange(0, [nextItems count])]]; + } + } + if (!ret && error != NULL) { + [*error retain]; + } + [pool drain]; + if (!ret && error != NULL) { + [*error autorelease]; + } + + return ret; +} +- (void)skipFile:(NSString *)thePath { + NSString *skipFilePath = [skipFilesRoot stringByAppendingString:thePath]; + NSError *myError = nil; + if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:skipFilePath targetUID:paramSet.targetUID targetGID:paramSet.targetGID error:&myError]) { + HSLogError(@"error creating parent dir for %@: %@", skipFilePath, myError); + return; + } + if (![[NSFileManager defaultManager] touchFileAtPath:skipFilePath targetUID:paramSet.targetUID targetGID:paramSet.targetGID error:&myError]) { + HSLogError(@"error touching %@: %@", skipFilePath, myError); + } + HSLogDebug(@"skip file %@", thePath); +} + +- (BOOL)requestMoreGlacierItems:(NSError **)error { + BOOL ret = YES; + NSAutoreleasePool *pool = nil; + while (bytesActuallyRequestedThisRound < bytesToRequestPerRound && [glacierRequestItems count] > 0) { + [pool drain]; + pool = [[NSAutoreleasePool alloc] init]; + GlacierRequestItem *item = [glacierRequestItems objectAtIndex:0]; + NSArray *nextItems = [item requestWithRestorer:self repo:repo error:error]; + if (nextItems == nil) { + ret = NO; + break; + } + [glacierRequestItems removeObjectAtIndex:0]; + if ([nextItems count] > 0) { + [glacierRequestItems insertObjects:nextItems atIndexes:[NSIndexSet indexSetWithIndexesInRange:NSMakeRange(0, [nextItems count])]]; + } + } + if (!ret && error != NULL) { + [*error retain]; + } + [pool drain]; + if (!ret && error != NULL) { + [*error autorelease]; + } + return ret; +} + + +- (BOOL)addToBytesRequested:(unsigned long long)length actualBytesRequested:(unsigned long long)actualBytesRequested error:(NSError **)error { + bytesRequested += length; + bytesActuallyRequestedThisRound += actualBytesRequested; + if ([delegate s3GlacierRestorerBytesRequestedDidChange:[NSNumber numberWithUnsignedLongLong:bytesRequested]]) { + SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested"); + return NO; + } + return YES; +} +- (BOOL)addToTotalBytesToRequest:(unsigned long long)length error:(NSError **)error { + totalBytesToRequest += length; + if ([delegate s3GlacierRestorerTotalBytesToRequestDidChange:[NSNumber numberWithUnsignedLongLong:totalBytesToRequest]]) { + SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested"); + return NO; + } + return YES; +} +- (BOOL)addToBytesTransferred:(unsigned long long)length error:(NSError **)error { + bytesTransferred += length; + if ([delegate s3GlacierRestorerBytesTransferredDidChange:[NSNumber numberWithUnsignedLongLong:bytesTransferred]]) { + SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested"); + return NO; + } + return YES; +} +- (BOOL)addToTotalBytesToTransfer:(unsigned long long)length error:(NSError **)error { + totalBytesToTransfer += length; + if ([delegate s3GlacierRestorerTotalBytesToTransferDidChange:[NSNumber numberWithUnsignedLongLong:totalBytesToTransfer]]) { + SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested"); + return NO; + } + return YES; +} + + +#pragma mark TargetConnectionDelegate +- (BOOL)targetConnectionShouldRetryOnTransientError:(NSError **)error { + return YES; +} +@end diff --git a/s3glacierrestore/S3GlacierRestorerDelegate.h b/s3glacierrestore/S3GlacierRestorerDelegate.h new file mode 100644 index 0000000..848eebd --- /dev/null +++ b/s3glacierrestore/S3GlacierRestorerDelegate.h @@ -0,0 +1,28 @@ +// +// S3GlacierRestorerDelegate.h +// Arq +// +// Created by Stefan Reitshamer on 1/9/14. +// Copyright (c) 2014 Stefan Reitshamer. All rights reserved. +// + + +@protocol S3GlacierRestorerDelegate + +// Methods return YES if cancel is requested. + +- (BOOL)s3GlacierRestorerMessageDidChange:(NSString *)message; + +- (BOOL)s3GlacierRestorerBytesRequestedDidChange:(NSNumber *)theRequested; +- (BOOL)s3GlacierRestorerTotalBytesToRequestDidChange:(NSNumber *)theMaxRequested; +- (BOOL)s3GlacierRestorerDidFinishRequesting; + +- (BOOL)s3GlacierRestorerBytesTransferredDidChange:(NSNumber *)theTransferred; +- (BOOL)s3GlacierRestorerTotalBytesToTransferDidChange:(NSNumber *)theTotal; + +- (BOOL)s3GlacierRestorerErrorMessage:(NSString *)theErrorMessage didOccurForPath:(NSString *)thePath; + +- (void)s3GlacierRestorerDidSucceed; +- (void)s3GlacierRestorerDidFail:(NSError *)error; + +@end diff --git a/s3glacierrestore/S3GlacierRestorerParamSet.h b/s3glacierrestore/S3GlacierRestorerParamSet.h new file mode 100644 index 0000000..bfe2493 --- /dev/null +++ b/s3glacierrestore/S3GlacierRestorerParamSet.h @@ -0,0 +1,61 @@ +// +// S3GlacierRestorerParamSet.h +// Arq +// +// Created by Stefan Reitshamer on 1/9/14. +// Copyright (c) 2014 Stefan Reitshamer. All rights reserved. +// + +@class BufferedInputStream; +@class BufferedOutputStream; +@class AWSRegion; +@class BlobKey; +@class Bucket; + + +@interface S3GlacierRestorerParamSet : NSObject { + Bucket *bucket; + NSString *encryptionPassword; + double downloadBytesPerSecond; + BlobKey *commitBlobKey; + NSString *rootItemName; + int treeVersion; + BOOL treeIsCompressed; + BlobKey *treeBlobKey; + NSString *nodeName; + uid_t targetUID; + gid_t targetGID; + BOOL useTargetUIDAndGID; + NSString *destinationPath; + int logLevel; +} + +- (id)initWithBucket:(Bucket *)theBucket + encryptionPassword:(NSString *)theEncryptionPassword +downloadBytesPerSecond:(double)theDownloadBytesPerSecond + commitBlobKey:(BlobKey *)theCommitBlobKey + rootItemName:(NSString *)theRootItemName + treeVersion:(int32_t)theTreeVersion + treeIsCompressed:(BOOL)theTreeIsCompressed + treeBlobKey:(BlobKey *)theTreeBlobKey + nodeName:(NSString *)theNodeName + targetUID:(uid_t)theTargetUID + targetGID:(gid_t)theTargetGID + useTargetUIDAndGID:(BOOL)theUseTargetUIDAndGID + destinationPath:(NSString *)theDestination + logLevel:(int)theLogLevel; + +@property (readonly, retain) Bucket *bucket; +@property (readonly, retain) NSString *encryptionPassword; +@property (readonly) double downloadBytesPerSecond; +@property (readonly, retain) BlobKey *commitBlobKey; +@property (readonly, retain) NSString *rootItemName; +@property (readonly, retain) BlobKey *treeBlobKey; +@property (readonly, retain) NSString *nodeName; +@property (readonly) uid_t targetUID; +@property (readonly) gid_t targetGID; +@property (readonly) BOOL useTargetUIDAndGID; +@property (readonly, retain) NSString *destinationPath; +@property (readonly) int logLevel; + +@end diff --git a/s3glacierrestore/S3GlacierRestorerParamSet.m b/s3glacierrestore/S3GlacierRestorerParamSet.m new file mode 100644 index 0000000..459fa85 --- /dev/null +++ b/s3glacierrestore/S3GlacierRestorerParamSet.m @@ -0,0 +1,79 @@ +// +// S3GlacierRestorerParamSet.m +// Arq +// +// Created by Stefan Reitshamer on 1/9/14. +// Copyright (c) 2014 Stefan Reitshamer. All rights reserved. +// + +#import "S3GlacierRestorerParamSet.h" +#import "AWSRegion.h" +#import "BlobKey.h" +#import "StringIO.h" +#import "IntegerIO.h" +#import "BlobKeyIO.h" +#import "BooleanIO.h" +#import "DataIO.h" +#import "Tree.h" +#import "DoubleIO.h" +#import "Bucket.h" + + +@implementation S3GlacierRestorerParamSet +@synthesize bucket; +@synthesize encryptionPassword; +@synthesize downloadBytesPerSecond; +@synthesize commitBlobKey; +@synthesize rootItemName; +@synthesize treeBlobKey; +@synthesize nodeName; +@synthesize targetUID; +@synthesize targetGID; +@synthesize useTargetUIDAndGID; +@synthesize destinationPath; +@synthesize logLevel; + + +- (id)initWithBucket:(Bucket *)theBucket + encryptionPassword:(NSString *)theEncryptionPassword +downloadBytesPerSecond:(double)theDownloadBytesPerSecond + commitBlobKey:(BlobKey *)theCommitBlobKey + rootItemName:(NSString *)theRootItemName + treeVersion:(int32_t)theTreeVersion + treeIsCompressed:(BOOL)theTreeIsCompressed + treeBlobKey:(BlobKey *)theTreeBlobKey + nodeName:(NSString *)theNodeName + targetUID:(uid_t)theTargetUID + targetGID:(gid_t)theTargetGID + useTargetUIDAndGID:(BOOL)theUseTargetUIDAndGID + destinationPath:(NSString *)theDestination + logLevel:(int)theLogLevel { + if (self = [super init]) { + bucket = [theBucket retain]; + encryptionPassword = [theEncryptionPassword retain]; + downloadBytesPerSecond = theDownloadBytesPerSecond; + commitBlobKey = [theCommitBlobKey retain]; + rootItemName = [theRootItemName retain]; + treeVersion = theTreeVersion; + treeIsCompressed = theTreeIsCompressed; + treeBlobKey = [theTreeBlobKey retain]; + nodeName = [theNodeName retain]; + targetUID = theTargetUID; + targetGID = theTargetGID; + useTargetUIDAndGID = theUseTargetUIDAndGID; + destinationPath = [theDestination retain]; + logLevel = theLogLevel; + } + return self; +} +- (void)dealloc { + [bucket release]; + [encryptionPassword release]; + [commitBlobKey release]; + [rootItemName release]; + [treeBlobKey release]; + [nodeName release]; + [destinationPath release]; + [super dealloc]; +} +@end