sjs/arq_restore
1
0
Fork 0
mirror of https://github.com/samsonjs/arq_restore.git synced 2026-04-27 15:07:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implemented restore from S3/Glacier and from (legacy) Glacier.

Browse source
This commit is contained in:
Stefan Reitshamer 2014-07-28 16:11:39 -04:00
parent 85a5b9976b
commit 20004756bc
24 changed files with 3049 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
ArqRestoreCommand.h
View file

@ -7,10 +7,12 @@
// //
#import "S3RestorerDelegate.h" #import "S3RestorerDelegate.h"
#import "S3GlacierRestorerDelegate.h"
#import "GlacierRestorerDelegate.h"
@class Target; @class Target;
@interface ArqRestoreCommand : NSObject <S3RestorerDelegate> { @interface ArqRestoreCommand : NSObject <S3RestorerDelegate, S3GlacierRestorerDelegate, GlacierRestorerDelegate> {
Target *target; Target *target;
} }

179
ArqRestoreCommand.m
View file

@ -19,6 +19,11 @@
#import "Commit.h" #import "Commit.h"
#import "BlobKey.h" #import "BlobKey.h"
#import "S3Restorer.h" #import "S3Restorer.h"
#import "S3GlacierRestorerParamSet.h"
#import "S3GlacierRestorer.h"
#import "GlacierRestorerParamSet.h"
#import "GlacierRestorer.h"
#import "S3AuthorizationProvider.h"
@implementation ArqRestoreCommand @implementation ArqRestoreCommand
@ -65,11 +70,11 @@
} }
targetParamsIndex += 2; targetParamsIndex += 2;
} else if ([cmd isEqualToString:@"restore"]) { } else if ([cmd isEqualToString:@"restore"]) {
if ((argc - targetParamsIndex) < 3) { if ((argc - targetParamsIndex) < 4) {
SETNSERROR([self errorDomain], ERROR_USAGE, @"missing arguments"); SETNSERROR([self errorDomain], ERROR_USAGE, @"missing arguments");
return NO; return NO;
} }
targetParamsIndex += 3; targetParamsIndex += 4;
} else { } else {
SETNSERROR([self errorDomain], ERROR_USAGE, @"unknown command: %@", cmd); SETNSERROR([self errorDomain], ERROR_USAGE, @"unknown command: %@", cmd);
return NO; return NO;
@ -93,7 +98,7 @@
return NO; return NO;
} }
} else if ([cmd isEqualToString:@"restore"]) { } else if ([cmd isEqualToString:@"restore"]) {
if (![self restoreComputerUUID:[args objectAtIndex:index+1] bucketUUID:[args objectAtIndex:index+3] encryptionPassword:[args objectAtIndex:index+2] error:error]) { if (![self restoreComputerUUID:[args objectAtIndex:index+1] bucketUUID:[args objectAtIndex:index+3] encryptionPassword:[args objectAtIndex:index+2] restoreBytesPerSecond:[args objectAtIndex:index+3] error:error]) {
return NO; return NO;
} }
} else { } else {
@ -201,7 +206,15 @@
} }
- (AWSRegion *)awsRegionForAccessKey:(NSString *)theAccessKey secretKey:(NSString *)theSecretKey bucketName:(NSString *)theBucketName error:(NSError **)error { - (AWSRegion *)awsRegionForAccessKey:(NSString *)theAccessKey secretKey:(NSString *)theSecretKey bucketName:(NSString *)theBucketName error:(NSError **)error {
return nil; S3AuthorizationProvider *sap = [[[S3AuthorizationProvider alloc] initWithAccessKey:theAccessKey secretKey:theSecretKey] autorelease];
NSURL *endpoint = [[AWSRegion usEast1] s3EndpointWithSSL:YES];
S3Service *s3 = [[[S3Service alloc] initWithS3AuthorizationProvider:sap endpoint:endpoint useAmazonRRS:NO] autorelease];
NSString *location = [s3 locationOfS3Bucket:theBucketName targetConnectionDelegate:nil error:error];
if (location == nil) {
return nil;
}
return [AWSRegion regionWithLocation:location];
} }
- (BOOL)listComputers:(NSError **)error { - (BOOL)listComputers:(NSError **)error {
@ -236,20 +249,20 @@
} }
- (NSArray *)expandedTargetList:(NSError **)error { - (NSArray *)expandedTargetList:(NSError **)error {
NSMutableArray *expandedTargetList = [NSMutableArray arrayWithObject:target]; NSMutableArray *expandedTargetList = [NSMutableArray arrayWithObject:target];
if ([target targetType] == kTargetAWS // if ([target targetType] == kTargetAWS
|| [target targetType] == kTargetDreamObjects // || [target targetType] == kTargetDreamObjects
|| [target targetType] == kTargetGoogleCloudStorage // || [target targetType] == kTargetGoogleCloudStorage
|| [target targetType] == kTargetGreenQloud // || [target targetType] == kTargetGreenQloud
|| [target targetType] == kTargetS3Compatible) { // || [target targetType] == kTargetS3Compatible) {
NSError *myError = nil; // NSError *myError = nil;
NSArray *targets = [self expandedTargetsForS3Target:target error:&myError]; // NSArray *targets = [self expandedTargetsForS3Target:target error:&myError];
if (targets == nil) { // if (targets == nil) {
HSLogError(@"failed to expand target list for %@: %@", target, myError); // HSLogError(@"failed to expand target list for %@: %@", target, myError);
} else { // } else {
[expandedTargetList setArray:targets]; // [expandedTargetList setArray:targets];
HSLogDebug(@"expandedTargetList is now: %@", expandedTargetList); // HSLogDebug(@"expandedTargetList is now: %@", expandedTargetList);
} // }
} // }
return expandedTargetList; return expandedTargetList;
} }
- (NSArray *)expandedTargetsForS3Target:(Target *)theTarget error:(NSError **)error { - (NSArray *)expandedTargetsForS3Target:(Target *)theTarget error:(NSError **)error {
@ -307,7 +320,7 @@
return YES; return YES;
} }
- (BOOL)restoreComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID encryptionPassword:(NSString *)theEncryptionPassword error:(NSError **)error { - (BOOL)restoreComputerUUID:(NSString *)theComputerUUID bucketUUID:(NSString *)theBucketUUID encryptionPassword:(NSString *)theEncryptionPassword restoreBytesPerSecond:(NSString *)theRestoreBytesPerSecond error:(NSError **)error {
Bucket *myBucket = nil; Bucket *myBucket = nil;
NSArray *expandedTargetList = [self expandedTargetList:error]; NSArray *expandedTargetList = [self expandedTargetList:error];
if (expandedTargetList == nil) { if (expandedTargetList == nil) {
@ -363,28 +376,51 @@
AWSRegion *region = [AWSRegion regionWithS3Endpoint:[target endpoint]]; AWSRegion *region = [AWSRegion regionWithS3Endpoint:[target endpoint]];
BOOL isGlacierDestination = [region supportsGlacier]; BOOL isGlacierDestination = [region supportsGlacier];
if ([myBucket storageType] == StorageTypeGlacier && isGlacierDestination) { if ([myBucket storageType] == StorageTypeGlacier && isGlacierDestination) {
// [[[GlacierRestoreController alloc] initWithAppConfig:appConfig int bytesPerSecond = [theRestoreBytesPerSecond intValue];
// doChownsAbove499:NO if (bytesPerSecond == 0) {
// destinationPath:destination SETNSERROR([self errorDomain], -1, @"invalid bytes_per_second %@", theRestoreBytesPerSecond);
// displayBucket:sourceOutlineController.selectedDisplayBucket return NO;
// displayCommit:sourceOutlineController.selectedDisplayCommit }
// displayNode:selectedNode
// mainWindow:mainWindow] autorelease]; GlacierRestorerParamSet *paramSet = [[[GlacierRestorerParamSet alloc] initWithBucket:myBucket
encryptionPassword:theEncryptionPassword
downloadBytesPerSecond:bytesPerSecond
commitBlobKey:commitBlobKey
rootItemName:[[myBucket localPath] lastPathComponent]
treeVersion:CURRENT_TREE_VERSION
treeIsCompressed:[[commit treeBlobKey] compressed]
treeBlobKey:[commit treeBlobKey]
nodeName:nil targetUID:getuid()
targetGID:getgid()
useTargetUIDAndGID:YES
destinationPath:destinationPath
logLevel:global_hslog_level] autorelease];
[[[GlacierRestorer alloc] initWithGlacierRestorerParamSet:paramSet delegate:self] autorelease];
} else if ([myBucket storageType] == StorageTypeS3Glacier && isGlacierDestination) { } else if ([myBucket storageType] == StorageTypeS3Glacier && isGlacierDestination) {
// [[[S3GlacierRestoreSetupController alloc] initWithLocalComputerUUID:[appConfig computerUUIDForTargetUUID:[target targetUUID]] int bytesPerSecond = [theRestoreBytesPerSecond intValue];
// doChownsAbove499:NO if (bytesPerSecond == 0) {
// destinationPath:destination SETNSERROR([self errorDomain], -1, @"invalid bytes_per_second %@", theRestoreBytesPerSecond);
// displayBucket:sourceOutlineController.selectedDisplayBucket return NO;
// displayCommit:sourceOutlineController.selectedDisplayCommit }
// displayNode:selectedNode
// mainWindow:mainWindow] autorelease]; S3GlacierRestorerParamSet *paramSet = [[[S3GlacierRestorerParamSet alloc] initWithBucket:myBucket
encryptionPassword:theEncryptionPassword
downloadBytesPerSecond:bytesPerSecond
commitBlobKey:commitBlobKey
rootItemName:[[myBucket localPath] lastPathComponent]
treeVersion:CURRENT_TREE_VERSION
treeIsCompressed:[[commit treeBlobKey] compressed]
treeBlobKey:[commit treeBlobKey]
nodeName:nil
targetUID:getuid()
targetGID:getgid()
useTargetUIDAndGID:YES
destinationPath:destinationPath
logLevel:global_hslog_level] autorelease];
[[[S3GlacierRestorer alloc] initWithS3GlacierRestorerParamSet:paramSet delegate:self] autorelease];
} else { } else {
// [[[S3RestoreController alloc] initWithAppConfig:appConfig
// doChownsAbove499:doChowns
// destinationPath:destination
// displayBucket:sourceOutlineController.selectedDisplayBucket
// displayCommit:sourceOutlineController.selectedDisplayCommit
// displayNode:selectedNode] autorelease];
S3RestorerParamSet *paramSet = [[[S3RestorerParamSet alloc] initWithBucket:myBucket S3RestorerParamSet *paramSet = [[[S3RestorerParamSet alloc] initWithBucket:myBucket
encryptionPassword:theEncryptionPassword encryptionPassword:theEncryptionPassword
commitBlobKey:commitBlobKey commitBlobKey:commitBlobKey
@ -429,4 +465,69 @@
printf("failed: %s\n", [[error localizedDescription] UTF8String]); printf("failed: %s\n", [[error localizedDescription] UTF8String]);
return NO; return NO;
} }
#pragma mark S3GlacierRestorerDelegate
- (BOOL)s3GlacierRestorerMessageDidChange:(NSString *)message {
printf("status: %s\n", [message UTF8String]);
return NO;
}
- (BOOL)s3GlacierRestorerBytesRequestedDidChange:(NSNumber *)theRequested {
return NO;
}
- (BOOL)s3GlacierRestorerTotalBytesToRequestDidChange:(NSNumber *)theMaxRequested {
return NO;
}
- (BOOL)s3GlacierRestorerDidFinishRequesting {
return NO;
}
- (BOOL)s3GlacierRestorerBytesTransferredDidChange:(NSNumber *)theTransferred {
return NO;
}
- (BOOL)s3GlacierRestorerTotalBytesToTransferDidChange:(NSNumber *)theTotal {
return NO;
}
- (BOOL)s3GlacierRestorerErrorMessage:(NSString *)theErrorMessage didOccurForPath:(NSString *)thePath {
printf("%s error: %s\n", [thePath UTF8String], [theErrorMessage UTF8String]);
return NO;
}
- (void)s3GlacierRestorerDidSucceed {
}
- (void)s3GlacierRestorerDidFail:(NSError *)error {
printf("failed: %s\n", [[error localizedDescription] UTF8String]);
}
#pragma mark GlacierRestorerDelegate
- (BOOL)glacierRestorerMessageDidChange:(NSString *)message {
printf("status: %s\n", [message UTF8String]);
return NO;
}
- (BOOL)glacierRestorerBytesRequestedDidChange:(NSNumber *)theRequested {
return NO;
}
- (BOOL)glacierRestorerTotalBytesToRequestDidChange:(NSNumber *)theMaxRequested {
return NO;
}
- (BOOL)glacierRestorerDidFinishRequesting {
return NO;
}
- (BOOL)glacierRestorerBytesTransferredDidChange:(NSNumber *)theTransferred {
return NO;
}
- (BOOL)glacierRestorerTotalBytesToTransferDidChange:(NSNumber *)theTotal {
return NO;
}
- (BOOL)glacierRestorerErrorMessage:(NSString *)theErrorMessage didOccurForPath:(NSString *)thePath {
printf("%s error: %s\n", [thePath UTF8String], [theErrorMessage UTF8String]);
return NO;
}
- (BOOL)glacierRestorerDidSucceed {
return NO;
}
- (BOOL)glacierRestorerDidFail:(NSError *)error {
printf("failed: %s\n", [[error localizedDescription] UTF8String]);
return NO;
}
@end @end

6
arq_restore.m
View file

@ -37,9 +37,9 @@
static void printUsage(const char *exeName) { static void printUsage(const char *exeName) {
fprintf(stderr, "Usage:\n"); fprintf(stderr, "Usage:\n");
fprintf(stderr, "\t%s [-l log_level] listcomputers <target_type> <target_params>\n", exeName); fprintf(stderr, "\t%s [-l log_level] listcomputers <target_type> <target_params>\n", exeName);
fprintf(stderr, "\t%s [-l log_level] listfolders <computer_uuid> <encryption_password> <target_type> <target_params>\n", exeName); fprintf(stderr, "\t%s [-l log_level] listfolders <computer_uuid> <encryption_password> <target_type> <target_params>\n", exeName);
fprintf(stderr, "\t%s [-l log_level] restore <computer_uuid> <encryption_password> <folder_uuid> <target_type> <target_params>\n", exeName); fprintf(stderr, "\t%s [-l log_level] restore <computer_uuid> <encryption_password> <folder_uuid> <bytes_per_second> <target_type> <target_params>\n", exeName);
fprintf(stderr, "\t\ntarget_params by target type:\n"); fprintf(stderr, "\t\ntarget_params by target type:\n");
fprintf(stderr, "\taws: access_key secret_key bucket_name\n"); fprintf(stderr, "\taws: access_key secret_key bucket_name\n");
fprintf(stderr, "\tsftp: hostname port path username password_or_keyfile [keyfile_passphrase]\n"); fprintf(stderr, "\tsftp: hostname port path username password_or_keyfile [keyfile_passphrase]\n");

74
arq_restore.xcodeproj/project.pbxproj
View file

@ -189,6 +189,15 @@
F8F2D9851986D3C400997A15 /* XAttrSet.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9841986D3C400997A15 /* XAttrSet.m */; }; F8F2D9851986D3C400997A15 /* XAttrSet.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9841986D3C400997A15 /* XAttrSet.m */; };
F8F2D9921986D4C700997A15 /* CalculateItem.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D98E1986D4C700997A15 /* CalculateItem.m */; }; F8F2D9921986D4C700997A15 /* CalculateItem.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D98E1986D4C700997A15 /* CalculateItem.m */; };
F8F2D9931986D4C700997A15 /* RestoreItem.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9901986D4C700997A15 /* RestoreItem.m */; }; F8F2D9931986D4C700997A15 /* RestoreItem.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9901986D4C700997A15 /* RestoreItem.m */; };
F8F2D9981986DCCC00997A15 /* S3GlacierRestorerParamSet.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9971986DCCC00997A15 /* S3GlacierRestorerParamSet.m */; };
F8F2D99C1986DDAD00997A15 /* S3GlacierRestorer.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D99A1986DDAD00997A15 /* S3GlacierRestorer.m */; };
F8F2D99F1986DE1800997A15 /* GlacierRequestItem.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D99E1986DE1800997A15 /* GlacierRequestItem.m */; };
F8F2D9A61986DE3B00997A15 /* GlacierPack.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9A11986DE3B00997A15 /* GlacierPack.m */; };
F8F2D9A71986DE3B00997A15 /* GlacierPackIndex.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9A31986DE3B00997A15 /* GlacierPackIndex.m */; };
F8F2D9A81986DE3B00997A15 /* GlacierPackSet.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9A51986DE3B00997A15 /* GlacierPackSet.m */; };
F8F2D9AB1986DE4400997A15 /* GlacierRestorer.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9AA1986DE4400997A15 /* GlacierRestorer.m */; };
F8F2D9AE1986DE8300997A15 /* BinarySHA1.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9AD1986DE8300997A15 /* BinarySHA1.m */; };
F8F2D9B11986DF6B00997A15 /* GlacierRestorerParamSet.m in Sources */ = {isa = PBXBuildFile; fileRef = F8F2D9B01986DF6B00997A15 /* GlacierRestorerParamSet.m */; };
/* End PBXBuildFile section */ /* End PBXBuildFile section */
/* Begin PBXCopyFilesBuildPhase section */ /* Begin PBXCopyFilesBuildPhase section */
@ -579,6 +588,26 @@
F8F2D98F1986D4C700997A15 /* RestoreItem.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RestoreItem.h; sourceTree = "<group>"; }; F8F2D98F1986D4C700997A15 /* RestoreItem.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RestoreItem.h; sourceTree = "<group>"; };
F8F2D9901986D4C700997A15 /* RestoreItem.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = RestoreItem.m; sourceTree = "<group>"; }; F8F2D9901986D4C700997A15 /* RestoreItem.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = RestoreItem.m; sourceTree = "<group>"; };
F8F2D9911986D4C700997A15 /* Restorer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Restorer.h; sourceTree = "<group>"; }; F8F2D9911986D4C700997A15 /* Restorer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Restorer.h; sourceTree = "<group>"; };
F8F2D9961986DCCC00997A15 /* S3GlacierRestorerParamSet.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = S3GlacierRestorerParamSet.h; sourceTree = "<group>"; };
F8F2D9971986DCCC00997A15 /* S3GlacierRestorerParamSet.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = S3GlacierRestorerParamSet.m; sourceTree = "<group>"; };
F8F2D9991986DDAC00997A15 /* S3GlacierRestorer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = S3GlacierRestorer.h; sourceTree = "<group>"; };
F8F2D99A1986DDAD00997A15 /* S3GlacierRestorer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = S3GlacierRestorer.m; sourceTree = "<group>"; };
F8F2D99B1986DDAD00997A15 /* S3GlacierRestorerDelegate.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = S3GlacierRestorerDelegate.h; sourceTree = "<group>"; };
F8F2D99D1986DE1800997A15 /* GlacierRequestItem.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = GlacierRequestItem.h; sourceTree = "<group>"; };
F8F2D99E1986DE1800997A15 /* GlacierRequestItem.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GlacierRequestItem.m; sourceTree = "<group>"; };
F8F2D9A01986DE3B00997A15 /* GlacierPack.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = GlacierPack.h; sourceTree = "<group>"; };
F8F2D9A11986DE3B00997A15 /* GlacierPack.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GlacierPack.m; sourceTree = "<group>"; };
F8F2D9A21986DE3B00997A15 /* GlacierPackIndex.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = GlacierPackIndex.h; sourceTree = "<group>"; };
F8F2D9A31986DE3B00997A15 /* GlacierPackIndex.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GlacierPackIndex.m; sourceTree = "<group>"; };
F8F2D9A41986DE3B00997A15 /* GlacierPackSet.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = GlacierPackSet.h; sourceTree = "<group>"; };
F8F2D9A51986DE3B00997A15 /* GlacierPackSet.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GlacierPackSet.m; sourceTree = "<group>"; };
F8F2D9A91986DE4400997A15 /* GlacierRestorer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = GlacierRestorer.h; sourceTree = "<group>"; };
F8F2D9AA1986DE4400997A15 /* GlacierRestorer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GlacierRestorer.m; sourceTree = "<group>"; };
F8F2D9AC1986DE8300997A15 /* BinarySHA1.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BinarySHA1.h; sourceTree = "<group>"; };
F8F2D9AD1986DE8300997A15 /* BinarySHA1.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BinarySHA1.m; sourceTree = "<group>"; };
F8F2D9AF1986DF6B00997A15 /* GlacierRestorerParamSet.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = GlacierRestorerParamSet.h; sourceTree = "<group>"; };
F8F2D9B01986DF6B00997A15 /* GlacierRestorerParamSet.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GlacierRestorerParamSet.m; sourceTree = "<group>"; };
F8F2D9B21986DFF700997A15 /* GlacierRestorerDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GlacierRestorerDelegate.h; sourceTree = "<group>"; };
/* End PBXFileReference section */ /* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */ /* Begin PBXFrameworksBuildPhase section */
@ -612,6 +641,8 @@
08FB7795FE84155DC02AAC07 /* arq_restore */, 08FB7795FE84155DC02AAC07 /* arq_restore */,
F8F2D98C1986D49600997A15 /* commonrestore */, F8F2D98C1986D49600997A15 /* commonrestore */,
F8F2D9711986D00C00997A15 /* s3restore */, F8F2D9711986D00C00997A15 /* s3restore */,
F8F2D9951986DCBD00997A15 /* s3glacierrestore */,
F8F2D9941986DCB100997A15 /* glacierrestore */,
08FB779DFE84155DC02AAC07 /* External Frameworks and Libraries */, 08FB779DFE84155DC02AAC07 /* External Frameworks and Libraries */,
1AB674ADFE9D54B511CA2CBB /* Products */, 1AB674ADFE9D54B511CA2CBB /* Products */,
F89A204513FAE29E0071D321 /* libz.dylib */, F89A204513FAE29E0071D321 /* libz.dylib */,
@ -1139,6 +1170,8 @@
F8F2D92B1986BA0700997A15 /* repo */ = { F8F2D92B1986BA0700997A15 /* repo */ = {
isa = PBXGroup; isa = PBXGroup;
children = ( children = (
F8F2D9AC1986DE8300997A15 /* BinarySHA1.h */,
F8F2D9AD1986DE8300997A15 /* BinarySHA1.m */,
F8F2D92C1986BA1400997A15 /* Commit.h */, F8F2D92C1986BA1400997A15 /* Commit.h */,
F8F2D92D1986BA1400997A15 /* Commit.m */, F8F2D92D1986BA1400997A15 /* Commit.m */,
F8F2D92F1986BA2200997A15 /* CommitFailedFile.h */, F8F2D92F1986BA2200997A15 /* CommitFailedFile.h */,
@ -1185,6 +1218,8 @@
children = ( children = (
F8F2D98D1986D4C700997A15 /* CalculateItem.h */, F8F2D98D1986D4C700997A15 /* CalculateItem.h */,
F8F2D98E1986D4C700997A15 /* CalculateItem.m */, F8F2D98E1986D4C700997A15 /* CalculateItem.m */,
F8F2D99D1986DE1800997A15 /* GlacierRequestItem.h */,
F8F2D99E1986DE1800997A15 /* GlacierRequestItem.m */,
F8F2D98F1986D4C700997A15 /* RestoreItem.h */, F8F2D98F1986D4C700997A15 /* RestoreItem.h */,
F8F2D9901986D4C700997A15 /* RestoreItem.m */, F8F2D9901986D4C700997A15 /* RestoreItem.m */,
F8F2D9911986D4C700997A15 /* Restorer.h */, F8F2D9911986D4C700997A15 /* Restorer.h */,
@ -1192,6 +1227,36 @@
path = commonrestore; path = commonrestore;
sourceTree = "<group>"; sourceTree = "<group>";
}; };
F8F2D9941986DCB100997A15 /* glacierrestore */ = {
isa = PBXGroup;
children = (
F8F2D9A01986DE3B00997A15 /* GlacierPack.h */,
F8F2D9A11986DE3B00997A15 /* GlacierPack.m */,
F8F2D9A21986DE3B00997A15 /* GlacierPackIndex.h */,
F8F2D9A31986DE3B00997A15 /* GlacierPackIndex.m */,
F8F2D9A41986DE3B00997A15 /* GlacierPackSet.h */,
F8F2D9A51986DE3B00997A15 /* GlacierPackSet.m */,
F8F2D9A91986DE4400997A15 /* GlacierRestorer.h */,
F8F2D9AA1986DE4400997A15 /* GlacierRestorer.m */,
F8F2D9B21986DFF700997A15 /* GlacierRestorerDelegate.h */,
F8F2D9AF1986DF6B00997A15 /* GlacierRestorerParamSet.h */,
F8F2D9B01986DF6B00997A15 /* GlacierRestorerParamSet.m */,
);
path = glacierrestore;
sourceTree = "<group>";
};
F8F2D9951986DCBD00997A15 /* s3glacierrestore */ = {
isa = PBXGroup;
children = (
F8F2D9991986DDAC00997A15 /* S3GlacierRestorer.h */,
F8F2D99A1986DDAD00997A15 /* S3GlacierRestorer.m */,
F8F2D99B1986DDAD00997A15 /* S3GlacierRestorerDelegate.h */,
F8F2D9961986DCCC00997A15 /* S3GlacierRestorerParamSet.h */,
F8F2D9971986DCCC00997A15 /* S3GlacierRestorerParamSet.m */,
);
path = s3glacierrestore;
sourceTree = "<group>";
};
/* End PBXGroup section */ /* End PBXGroup section */
/* Begin PBXNativeTarget section */ /* Begin PBXNativeTarget section */
@ -1250,6 +1315,7 @@
F8F2D8821986B63400997A15 /* BaseTargetConnection.m in Sources */, F8F2D8821986B63400997A15 /* BaseTargetConnection.m in Sources */,
F8F2D9931986D4C700997A15 /* RestoreItem.m in Sources */, F8F2D9931986D4C700997A15 /* RestoreItem.m in Sources */,
F8F2D93D1986BA7900997A15 /* UserLibrary.m in Sources */, F8F2D93D1986BA7900997A15 /* UserLibrary.m in Sources */,
F8F2D99F1986DE1800997A15 /* GlacierRequestItem.m in Sources */,
F8F2D86E1986B5CC00997A15 /* SFTPTargetConnection.m in Sources */, F8F2D86E1986B5CC00997A15 /* SFTPTargetConnection.m in Sources */,
F829522019868E26001DC91B /* GoogleDrive.m in Sources */, F829522019868E26001DC91B /* GoogleDrive.m in Sources */,
F8F2D9001986B78600997A15 /* BooleanNode.m in Sources */, F8F2D9001986B78600997A15 /* BooleanNode.m in Sources */,
@ -1292,6 +1358,8 @@
F829DC10198691CB00D637E0 /* BooleanIO.m in Sources */, F829DC10198691CB00D637E0 /* BooleanIO.m in Sources */,
F8F2D9491986BAD200997A15 /* Repo.m in Sources */, F8F2D9491986BAD200997A15 /* Repo.m in Sources */,
F829522119868E26001DC91B /* GoogleDriveErrorResult.m in Sources */, F829522119868E26001DC91B /* GoogleDriveErrorResult.m in Sources */,
F8F2D99C1986DDAD00997A15 /* S3GlacierRestorer.m in Sources */,
F8F2D9B11986DF6B00997A15 /* GlacierRestorerParamSet.m in Sources */,
F8295167198683D5001DC91B /* scp.c in Sources */, F8295167198683D5001DC91B /* scp.c in Sources */,
F8F2D9531986BE2A00997A15 /* FarkImpl.m in Sources */, F8F2D9531986BE2A00997A15 /* FarkImpl.m in Sources */,
F829DC241986924100D637E0 /* FDInputStream.m in Sources */, F829DC241986924100D637E0 /* FDInputStream.m in Sources */,
@ -1312,7 +1380,9 @@
F8F2D9771986D32B00997A15 /* S3Restorer.m in Sources */, F8F2D9771986D32B00997A15 /* S3Restorer.m in Sources */,
F8F2D8931986B66000997A15 /* GlacierJobLister.m in Sources */, F8F2D8931986B66000997A15 /* GlacierJobLister.m in Sources */,
F8F2D9031986B78600997A15 /* RealNode.m in Sources */, F8F2D9031986B78600997A15 /* RealNode.m in Sources */,
F8F2D9A71986DE3B00997A15 /* GlacierPackIndex.m in Sources */,
F8F2D87C1986B61800997A15 /* GoogleDriveRemoteFS.m in Sources */, F8F2D87C1986B61800997A15 /* GoogleDriveRemoteFS.m in Sources */,
F8F2D9981986DCCC00997A15 /* S3GlacierRestorerParamSet.m in Sources */,
F829521319868DE6001DC91B /* RegexKitLite.m in Sources */, F829521319868DE6001DC91B /* RegexKitLite.m in Sources */,
F8F2D8FD1986B78600997A15 /* ArrayNode.m in Sources */, F8F2D8FD1986B78600997A15 /* ArrayNode.m in Sources */,
F8F2D9741986D02D00997A15 /* S3RestorerParamSet.m in Sources */, F8F2D9741986D02D00997A15 /* S3RestorerParamSet.m in Sources */,
@ -1320,6 +1390,7 @@
F8F2D9251986B98600997A15 /* BlobKey.m in Sources */, F8F2D9251986B98600997A15 /* BlobKey.m in Sources */,
F829523319868E59001DC91B /* SBJsonParser.m in Sources */, F829523319868E59001DC91B /* SBJsonParser.m in Sources */,
F8F2D8A51986B67500997A15 /* Vault.m in Sources */, F8F2D8A51986B67500997A15 /* Vault.m in Sources */,
F8F2D9A61986DE3B00997A15 /* GlacierPack.m in Sources */,
F8F2D97F1986D3A100997A15 /* FileAttributes.m in Sources */, F8F2D97F1986D3A100997A15 /* FileAttributes.m in Sources */,
F829DC15198691CB00D637E0 /* NSErrorIO.m in Sources */, F829DC15198691CB00D637E0 /* NSErrorIO.m in Sources */,
F82951EE19868D90001DC91B /* HTTPInputStream.m in Sources */, F82951EE19868D90001DC91B /* HTTPInputStream.m in Sources */,
@ -1381,6 +1452,7 @@
F8F2D8941986B66000997A15 /* GlacierRequest.m in Sources */, F8F2D8941986B66000997A15 /* GlacierRequest.m in Sources */,
F829DBFF1986901300D637E0 /* Streams.m in Sources */, F829DBFF1986901300D637E0 /* Streams.m in Sources */,
F8F2D8A41986B67500997A15 /* SHA256TreeHash.m in Sources */, F8F2D8A41986B67500997A15 /* SHA256TreeHash.m in Sources */,
F8F2D9A81986DE3B00997A15 /* GlacierPackSet.m in Sources */,
F8F2D9071986B78600997A15 /* XMLPListWriter.m in Sources */, F8F2D9071986B78600997A15 /* XMLPListWriter.m in Sources */,
F829DC1C1986921E00D637E0 /* MD5Hash.m in Sources */, F829DC1C1986921E00D637E0 /* MD5Hash.m in Sources */,
F8F2D8681986B58000997A15 /* BackupSet.m in Sources */, F8F2D8681986B58000997A15 /* BackupSet.m in Sources */,
@ -1408,9 +1480,11 @@
F8F2D8AD1986B6A300997A15 /* SHA256Hash.m in Sources */, F8F2D8AD1986B6A300997A15 /* SHA256Hash.m in Sources */,
F8295168198683D5001DC91B /* session.c in Sources */, F8295168198683D5001DC91B /* session.c in Sources */,
F8F2D86D1986B5CC00997A15 /* S3TargetConnection.m in Sources */, F8F2D86D1986B5CC00997A15 /* S3TargetConnection.m in Sources */,
F8F2D9AB1986DE4400997A15 /* GlacierRestorer.m in Sources */,
F83F9B2D1983303F007CBFB4 /* ArqRestoreCommand.m in Sources */, F83F9B2D1983303F007CBFB4 /* ArqRestoreCommand.m in Sources */,
F8F2D9341986BA2B00997A15 /* ArqSalt.m in Sources */, F8F2D9341986BA2B00997A15 /* ArqSalt.m in Sources */,
F829DC14198691CB00D637E0 /* IntegerIO.m in Sources */, F829DC14198691CB00D637E0 /* IntegerIO.m in Sources */,
F8F2D9AE1986DE8300997A15 /* BinarySHA1.m in Sources */,
F8F2D8FF1986B78600997A15 /* BinaryPListWriter.m in Sources */, F8F2D8FF1986B78600997A15 /* BinaryPListWriter.m in Sources */,
F8F2D95B1986BE4E00997A15 /* PackIndex.m in Sources */, F8F2D95B1986BE4E00997A15 /* PackIndex.m in Sources */,
F829522319868E26001DC91B /* GoogleDriveRequest.m in Sources */, F829522319868E26001DC91B /* GoogleDriveRequest.m in Sources */,

26
commonrestore/GlacierRequestItem.h Normal file
View file

@ -0,0 +1,26 @@
//
// GlacierRequestItem.h
// Arq
//
// Created by Stefan Reitshamer on 5/30/13.
// Copyright (c) 2013 Stefan Reitshamer. All rights reserved.
//
@class Tree;
@class Node;
@protocol Restorer;
@class Repo;
@interface GlacierRequestItem : NSObject {
Tree *tree;
Node *node;
NSString *path;
BOOL requestedFirstBlobKey;
NSUInteger dataBlobKeyIndex;
}
- (id)initWithPath:(NSString *)thePath tree:(Tree *)theTree;
- (id)initWithPath:(NSString *)thePath node:(Node *)theNode;
- (NSArray *)requestWithRestorer:(id <Restorer>)theRestorer repo:(Repo *)theRepo error:(NSError **)error;
@end

106
commonrestore/GlacierRequestItem.m Normal file
View file

@ -0,0 +1,106 @@
//
// GlacierRequestItem.m
// Arq
//
// Created by Stefan Reitshamer on 5/30/13.
// Copyright (c) 2013 Stefan Reitshamer. All rights reserved.
//
#import "GlacierRequestItem.h"
#import "Tree.h"
#import "Node.h"
#import "GlacierRestorer.h"
#import "BlobKey.h"
#import "Repo.h"
@implementation GlacierRequestItem
- (id)initWithPath:(NSString *)thePath tree:(Tree *)theTree {
if (self = [super init]) {
path = [thePath retain];
tree = [theTree retain];
}
return self;
}
- (id)initWithPath:(NSString *)thePath node:(Node *)theNode {
if (self = [super init]) {
path = [thePath retain];
node = [theNode retain];
}
return self;
}
- (id)initWithPath:(NSString *)thePath node:(Node *)theNode dataBlobKeyIndex:(NSUInteger)theDataBlobKeyIndex {
if (self = [super init]) {
path = [thePath retain];
node = [theNode retain];
dataBlobKeyIndex = theDataBlobKeyIndex;
NSAssert(theDataBlobKeyIndex > 0, @"theDataBlobKeyIndex must be > 0");
requestedFirstBlobKey = YES;
}
return self;
}
- (void)dealloc {
[path release];
[tree release];
[node release];
[super dealloc];
}
- (NSArray *)requestWithRestorer:(id <Restorer>)theRestorer repo:(Repo *)theRepo error:(NSError **)error {
NSMutableArray *nextItems = [NSMutableArray array];
if (tree != nil) {
if (![theRestorer requestBlobKey:[tree xattrsBlobKey] error:error]) {
return nil;
}
if (![theRestorer requestBlobKey:[tree aclBlobKey] error:error]) {
return nil;
}
for (NSString *childNodeName in [tree childNodeNames]) {
Node *childNode = [tree childNodeWithName:childNodeName];
NSString *childPath = [path stringByAppendingPathComponent:childNodeName];
[nextItems addObject:[[[GlacierRequestItem alloc] initWithPath:childPath node:childNode] autorelease]];
}
} else {
NSAssert(node != nil, @"node can't be nil if tree is nil");
if ([node isTree]) {
Tree *childTree = [theRepo treeForBlobKey:[node treeBlobKey] error:error];
if (childTree == nil) {
return nil;
}
[nextItems addObject:[[[GlacierRequestItem alloc] initWithPath:path tree:childTree] autorelease]];
} else {
if (!requestedFirstBlobKey) {
if (![theRestorer requestBlobKey:[node xattrsBlobKey] error:error]) {
return nil;
}
if (![theRestorer requestBlobKey:[node aclBlobKey] error:error]) {
return nil;
}
if (![theRestorer shouldSkipFile:path]) {
if ([[node dataBlobKeys] count] > 0) {
BlobKey *firstKey = [[node dataBlobKeys] objectAtIndex:0];
HSLogDetail(@"requesting first data blob of %ld for %@", (unsigned long)[[node dataBlobKeys] count], path);
if (![theRestorer requestBlobKey:firstKey error:error]) {
return nil;
}
if ([[node dataBlobKeys] count] > 1) {
[nextItems addObject:[[[GlacierRequestItem alloc] initWithPath:path node:node dataBlobKeyIndex:1] autorelease]];
}
}
}
} else {
BlobKey *blobKey = [[node dataBlobKeys] objectAtIndex:dataBlobKeyIndex];
HSLogDetail(@"requesting data blob %ld of %ld for %@", (unsigned long)(dataBlobKeyIndex + 1), (unsigned long)[[node dataBlobKeys] count], path);
if (![theRestorer requestBlobKey:blobKey error:error]) {
return nil;
}
if ([[node dataBlobKeys] count] > (dataBlobKeyIndex + 1)) {
[nextItems addObject:[[[GlacierRequestItem alloc] initWithPath:path node:node dataBlobKeyIndex:(dataBlobKeyIndex + 1)] autorelease]];
}
}
}
}
return nextItems;
}
@end

38
glacierrestore/GlacierPack.h Normal file
View file

@ -0,0 +1,38 @@
//
// GlacierPack.h
//
// Created by Stefan Reitshamer on 11/3/12.
// Copyright (c) 2012 Stefan Reitshamer. All rights reserved.
//
@class Target;
@interface GlacierPack : NSObject {
NSString *s3BucketName;
NSString *computerUUID;
NSString *bucketUUID;
NSString *packSetName;
NSString *packSHA1;
NSString *archiveId;
NSString *localPath;
unsigned long long packSize;
uid_t uid;
gid_t gid;
}
- (id)initWithTarget:(Target *)theTarget
s3BucketName:(NSString *)theS3BucketName
computerUUID:(NSString *)theComputerUUID
bucketUUID:(NSString *)theBucketUUID
packSHA1:(NSString *)thePackSHA1
archiveId:(NSString *)theArchiveId
packSize:(unsigned long long)thePackSize
targetUID:(uid_t)theTargetUID
targetGID:(gid_t)theTargetGID;
- (NSString *)packSHA1;
- (NSString *)archiveId;
- (unsigned long long)packSize;
- (BOOL)cachePackDataToDisk:(NSData *)thePackData error:(NSError **)error;
- (NSData *)cachedDataForObjectAtOffset:(unsigned long long)offset error:(NSError **)error;
@end

121
glacierrestore/GlacierPack.m Normal file
View file

@ -0,0 +1,121 @@
//
// GlacierPack.m
//
// Created by Stefan Reitshamer on 11/3/12.
// Copyright (c) 2012 Stefan Reitshamer. All rights reserved.
//
#import "GlacierPack.h"
#import "UserLibrary_Arq.h"
#import "NSFileManager_extra.h"
#import "FDInputStream.h"
#import "BufferedInputStream.h"
#import "StringIO.h"
#import "IntegerIO.h"
#import "Streams.h"
#import "Target.h"
@implementation GlacierPack
- (id)initWithTarget:(Target *)theTarget
s3BucketName:(NSString *)theS3BucketName
computerUUID:(NSString *)theComputerUUID
bucketUUID:(NSString *)theBucketUUID
packSHA1:(NSString *)thePackSHA1
archiveId:(NSString *)theArchiveId
packSize:(unsigned long long)thePackSize
targetUID:(uid_t)theTargetUID
targetGID:(gid_t)theTargetGID {
if (self = [super init]) {
s3BucketName = [theS3BucketName retain];
computerUUID = [theComputerUUID retain];
bucketUUID = [theBucketUUID retain];
packSetName = [[NSString alloc] initWithFormat:@"%@-glacierblobs", theBucketUUID];
packSHA1 = [thePackSHA1 retain];
archiveId = [theArchiveId retain];
packSize = thePackSize;
uid = theTargetUID;
gid = theTargetGID;
localPath = [[NSString alloc] initWithFormat:@"%@/%@/%@/glacier_packsets/%@/%@/%@.pack",
[UserLibrary arqCachePath], [theTarget targetUUID], computerUUID, packSetName, [packSHA1 substringToIndex:2], [packSHA1 substringFromIndex:2]];
}
return self;
}
- (void)dealloc {
[s3BucketName release];
[computerUUID release];
[bucketUUID release];
[packSetName release];
[packSHA1 release];
[archiveId release];
[localPath release];
[super dealloc];
}
- (NSString *)packSHA1 {
return packSHA1;
}
- (NSString *)archiveId {
return archiveId;
}
- (unsigned long long)packSize {
return packSize;
}
- (BOOL)cachePackDataToDisk:(NSData *)thePackData error:(NSError **)error {
if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:localPath targetUID:uid targetGID:gid error:error]) {
return NO;
}
return [Streams writeData:thePackData atomicallyToFile:localPath targetUID:uid targetGID:gid bytesWritten:NULL error:error];
}
- (NSData *)cachedDataForObjectAtOffset:(unsigned long long)offset error:(NSError **)error {
int fd = open([localPath fileSystemRepresentation], O_RDONLY);
if (fd == -1) {
int errnum = errno;
HSLogError(@"open(%@) error %d: %s", localPath, errnum, strerror(errnum));
SETNSERROR(@"UnixErrorDomain", errnum, @"failed to open %@: %s", localPath, strerror(errnum));
return nil;
}
NSData *ret = nil;
FDInputStream *fdis = [[FDInputStream alloc] initWithFD:fd label:localPath];
BufferedInputStream *bis = [[BufferedInputStream alloc] initWithUnderlyingStream:fdis];
do {
if (lseek(fd, offset, SEEK_SET) == -1) {
int errnum = errno;
HSLogError(@"lseek(%@, %qu) error %d: %s", localPath, offset, errnum, strerror(errnum));
SETNSERROR(@"UnixErrorDomain", errnum, @"failed to seek to %qu in %@: %s", offset, localPath, strerror(errnum));
break;
}
NSString *mimeType;
NSString *downloadName;
if (![StringIO read:&mimeType from:bis error:error] || ![StringIO read:&downloadName from:bis error:error]) {
break;
}
uint64_t dataLen = 0;
if (![IntegerIO readUInt64:&dataLen from:bis error:error]) {
break;
}
NSData *data = nil;
if (dataLen > 0) {
unsigned char *buf = (unsigned char *)malloc((size_t)dataLen);
if (![bis readExactly:(NSUInteger)dataLen into:buf error:error]) {
free(buf);
break;
}
data = [NSData dataWithBytesNoCopy:buf length:(NSUInteger)dataLen];
} else {
data = [NSData data];
}
ret = data;
} while (0);
close(fd);
[bis release];
[fdis release];
return ret;
}
#pragma mark NSObject
- (NSString *)description {
return [NSString stringWithFormat:@"<GlacierPack packSHA1=%@>", packSHA1];
}
@end

53
glacierrestore/GlacierPackIndex.h Normal file
View file

@ -0,0 +1,53 @@
//
// GlacierPackIndex.h
//
// Created by Stefan Reitshamer on 11/3/12.
// Copyright (c) 2012 Stefan Reitshamer. All rights reserved.
//
@class S3Service;
@class PackIndexEntry;
@class PackId;
@class Target;
@protocol TargetConnectionDelegate;
@interface GlacierPackIndex : NSObject {
S3Service *s3;
NSString *s3BucketName;
NSString *computerUUID;
PackId *packId;
NSString *s3Path;
NSString *localPath;
uid_t targetUID;
gid_t targetGID;
NSMutableArray *pies;
NSString *archiveId;
unsigned long long packSize;
}
+ (NSString *)s3PathWithS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID packId:(PackId *)thePackId;
+ (NSString *)localPathWithTarget:(Target *)theTarget computerUUID:(NSString *)theComputerUUID packId:(PackId *)thePackId;
+ (NSArray *)glacierPackIndexesForTarget:(Target *)theTarget
s3Service:(S3Service *)theS3
s3BucketName:theS3BucketName
computerUUID:(NSString *)theComputerUUID
packSetName:(NSString *)thePackSetName
targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD
targetUID:(uid_t)theTargetUID
targetGID:(gid_t)theTargetGID
error:(NSError **)error;
- (id)initWithTarget:(Target *)theTarget
s3Service:(S3Service *)theS3
s3BucketName:(NSString *)theS3BucketName
computerUUID:(NSString *)theComputerUUID
packId:(PackId *)thePackId
targetUID:(uid_t)theTargetUID
targetGID:(gid_t)theTargetGID;
- (BOOL)makeLocalWithTargetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error;
- (NSArray *)allPackIndexEntriesWithTargetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error;
- (PackIndexEntry *)entryForSHA1:(NSString *)sha1 error:(NSError **)error;
- (PackId *)packId;
- (NSString *)archiveId:(NSError **)error;
- (unsigned long long)packSize:(NSError **)error;
@end

405
glacierrestore/GlacierPackIndex.m Normal file
View file

@ -0,0 +1,405 @@
//
// GlacierPackIndex.m
//
// Created by Stefan Reitshamer on 11/3/12.
// Copyright (c) 2012 Stefan Reitshamer. All rights reserved.
//
#include <sys/stat.h>
#include <sys/mman.h>
#include <libkern/OSByteOrder.h>
#import "GlacierPackIndex.h"
#import "S3Service.h"
#import "RegexKitLite.h"
#import "NSString_extra.h"
#import "BinarySHA1.h"
#import "PackIndexEntry.h"
#import "FileOutputStream.h"
#import "Streams.h"
#import "NSFileManager_extra.h"
#import "UserLibrary_Arq.h"
#import "NSError_extra.h"
#import "DataInputStream.h"
#import "FDInputStream.h"
#import "BufferedInputStream.h"
#import "StringIO.h"
#import "IntegerIO.h"
#import "PackId.h"
#import "Target.h"
typedef struct index_object {
uint64_t nbo_offset;
uint64_t nbo_datalength;
unsigned char sha1[20];
unsigned char filler[4];
} index_object;
typedef struct pack_index {
uint32_t magic_number;
uint32_t nbo_version;
uint32_t nbo_fanout[256];
index_object first_index_object;
} pack_index;
@implementation GlacierPackIndex
+ (NSString *)s3PathWithS3BucketName:(NSString *)theS3BucketName computerUUID:(NSString *)theComputerUUID packId:(PackId *)thePackId {
return [NSString stringWithFormat:@"/%@/%@/packsets/%@/%@.index", theS3BucketName, theComputerUUID, [thePackId packSetName], [thePackId packSHA1]];
}
+ (NSString *)localPathWithTarget:(Target *)theTarget computerUUID:(NSString *)theComputerUUID packId:(PackId *)thePackId {
return [NSString stringWithFormat:@"%@/%@/%@/glacier_packsets/%@/%@/%@.index", [UserLibrary arqCachePath], [theTarget targetUUID], theComputerUUID, [thePackId packSetName], [[thePackId packSHA1] substringToIndex:2], [[thePackId packSHA1] substringFromIndex:2]];
}
+ (NSArray *)glacierPackIndexesForTarget:(Target *)theTarget
s3Service:(S3Service *)theS3
s3BucketName:theS3BucketName
computerUUID:(NSString *)theComputerUUID
packSetName:(NSString *)thePackSetName
targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD
targetUID:(uid_t)theTargetUID
targetGID:(gid_t)theTargetGID
error:(NSError **)error {
NSMutableArray *diskPackIndexes = [NSMutableArray array];
NSString *packSetsPrefix = [NSString stringWithFormat:@"/%@/%@/packsets/%@/", theS3BucketName, theComputerUUID, thePackSetName];
NSArray *paths = [theS3 pathsWithPrefix:packSetsPrefix targetConnectionDelegate:theTCD error:error];
if (paths == nil) {
return nil;
}
for (NSString *thePath in paths) {
NSRange sha1Range = [thePath rangeOfRegex:@"/(\\w+)\\.index$" capture:1];
if (sha1Range.location != NSNotFound) {
NSString *thePackSHA1 = [thePath substringWithRange:sha1Range];
PackId *packId = [[[PackId alloc] initWithPackSetName:thePackSetName packSHA1:thePackSHA1] autorelease];
GlacierPackIndex *index = [[GlacierPackIndex alloc] initWithTarget:theTarget
s3Service:theS3
s3BucketName:theS3BucketName
computerUUID:theComputerUUID
packId:packId
targetUID:theTargetUID
targetGID:theTargetGID];
[diskPackIndexes addObject:index];
[index release];
}
}
return diskPackIndexes;
}
- (id)initWithTarget:(Target *)theTarget
s3Service:(S3Service *)theS3
s3BucketName:(NSString *)theS3BucketName
computerUUID:(NSString *)theComputerUUID
packId:(PackId *)thePackId
targetUID:(uid_t)theTargetUID
targetGID:(gid_t)theTargetGID {
if (self = [super init]) {
s3 = [theS3 retain];
s3BucketName = [theS3BucketName retain];
computerUUID = [theComputerUUID retain];
packId = [thePackId retain];
s3Path = [[GlacierPackIndex s3PathWithS3BucketName:s3BucketName computerUUID:computerUUID packId:packId] retain];
localPath = [[GlacierPackIndex localPathWithTarget:theTarget computerUUID:computerUUID packId:packId] retain];
targetUID = theTargetUID;
targetGID = theTargetGID;
}
return self;
}
- (void)dealloc {
[s3 release];
[s3BucketName release];
[computerUUID release];
[packId release];
[s3Path release];
[localPath release];
[pies release];
[archiveId release];
[super dealloc];
}
- (BOOL)makeLocalWithTargetConnectionDelegate:(id)theTCD error:(NSError **)error {
NSFileManager *fm = [NSFileManager defaultManager];
BOOL ret = YES;
if (![fm fileExistsAtPath:localPath]) {
for (;;) {
HSLogDebug(@"packset %@: making pack index %@ local", packId, packId);
NSError *myError = nil;
NSData *data = [s3 dataAtPath:s3Path targetConnectionDelegate:theTCD error:&myError];
if (data != nil) {
ret = [self savePackIndex:data error:error];
break;
}
if (![myError isTransientError]) {
HSLogError(@"error getting S3 pack index %@: %@", s3Path, myError);
if (error != NULL) {
*error = myError;
}
ret = NO;
break;
} else {
HSLogWarn(@"network error making pack index %@ local (retrying): %@", s3Path, myError);
NSError *rmError = nil;
if ([[NSFileManager defaultManager] fileExistsAtPath:localPath] && ![[NSFileManager defaultManager] removeItemAtPath:localPath error:&rmError]) {
HSLogError(@"error deleting incomplete downloaded pack index %@: %@", localPath, rmError);
}
}
}
}
return ret;
}
- (NSArray *)allPackIndexEntriesWithTargetConnectionDelegate:(id<TargetConnectionDelegate>)theTCD error:(NSError **)error {
if (![self makeLocalWithTargetConnectionDelegate:theTCD error:error]) {
return nil;
}
if (![self readFile:error]) {
return nil;
}
return pies;
}
- (PackIndexEntry *)entryForSHA1:(NSString *)sha1 error:(NSError **)error {
if (error != NULL) {
*error = nil;
}
NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
PackIndexEntry *ret = [self doEntryForSHA1:sha1 error:(NSError **)error];
[ret retain];
if (ret == nil && error != NULL) {
[*error retain];
}
[pool drain];
[ret autorelease];
if (ret == nil && error != NULL) {
[*error autorelease];
}
return ret;
}
- (PackId *)packId {
return packId;
}
- (NSString *)archiveId:(NSError **)error {
if (![self readFile:error]) {
return nil;
}
return archiveId;
}
- (unsigned long long)packSize:(NSError **)error {
if (![self readFile:error]) {
return 0;
}
return packSize;
}
#pragma mark NSObject
- (NSString *)description {
return [NSString stringWithFormat:@"<DiskPackIndex: computerUUID=%@ packId=%@>", computerUUID, packId];
}
#pragma mark internal
- (BOOL)readFile:(NSError **)error {
if (pies != nil) {
return YES;
}
int fd = open([localPath fileSystemRepresentation], O_RDONLY);
if (fd == -1) {
int errnum = errno;
HSLogError(@"open(%@) error %d: %s", localPath, errnum, strerror(errnum));
SETNSERROR(@"UnixErrorDomain", errnum, @"failed to open %@: %s", localPath, strerror(errnum));
return NO;
}
struct stat st;
if (fstat(fd, &st) == -1) {
int errnum = errno;
HSLogError(@"fstat(%@) error %d: %s", localPath, errnum, strerror(errnum));
SETNSERROR(@"UnixErrorDomain", errnum, @"%@: %s", localPath, strerror(errnum));
close(fd);
return NO;
}
if (st.st_size < sizeof(pack_index)) {
HSLogError(@"pack index length %ld is less than size of pack_index", (unsigned long)st.st_size);
SETNSERROR(@"GlacierPackIndexErrorDomain", -1, @"pack index length is less than size of pack_index");
close(fd);
return NO;
}
pack_index *the_pack_index = mmap(0, (size_t)st.st_size, PROT_READ, MAP_SHARED, fd, 0);
if (the_pack_index == MAP_FAILED) {
int errnum = errno;
HSLogError(@"mmap(%@) error %d: %s", localPath, errnum, strerror(errnum));
SETNSERROR(@"UnixErrorDomain", errnum, @"error mapping %@ to memory: %s", localPath, strerror(errnum));
close(fd);
return NO;
}
pies = [[NSMutableArray alloc] init];
uint32_t count = OSSwapBigToHostInt32(the_pack_index->nbo_fanout[255]);
index_object *indexObjects = &(the_pack_index->first_index_object);
for (uint32_t i = 0; i < count; i++) {
NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
uint64_t offset = OSSwapBigToHostInt64(indexObjects[i].nbo_offset);
uint64_t dataLength = OSSwapBigToHostInt64(indexObjects[i].nbo_datalength);
NSString *objectSHA1 = [NSString hexStringWithBytes:indexObjects[i].sha1 length:20];
PackIndexEntry *pie = [[[PackIndexEntry alloc] initWithPackId:packId offset:offset dataLength:dataLength objectSHA1:objectSHA1] autorelease];
[pies addObject:pie];
[pool drain];
}
if (munmap(the_pack_index, (size_t)st.st_size) == -1) {
int errnum = errno;
HSLogError(@"munmap: %s", strerror(errnum));
}
uint32_t offset = sizeof(pack_index) + (count - 1) * sizeof(index_object);
if (!lseek(fd, offset, SEEK_SET) == -1) {
int errnum = errno;
HSLogError(@"lstat(%@, %ld) error %d: %s", localPath, (unsigned long)offset, errnum, strerror(errnum));
SETNSERROR(@"UnixErrorDomain", errnum, @"error seeking to archiveId in pack index file");
close(fd);
return NO;
}
FDInputStream *fdis = [[[FDInputStream alloc] initWithFD:fd label:@"packindex"] autorelease];
BufferedInputStream *bis = [[[BufferedInputStream alloc] initWithUnderlyingStream:fdis] autorelease];
BOOL ret = [StringIO read:&archiveId from:bis error:error] && [IntegerIO readUInt64:&packSize from:bis error:error];
[archiveId retain];
close(fd);
if (!ret) {
return NO;
}
return YES;
}
- (BOOL)savePackIndex:(NSData *)theData error:(NSError **)error {
if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:localPath targetUID:targetUID targetGID:targetGID error:error]) {
return NO;
}
id <InputStream> is = [[[DataInputStream alloc] initWithData:theData description:[self description]] autorelease];
NSError *myError = nil;
unsigned long long written = 0;
BOOL ret = [Streams transferFrom:is atomicallyToFile:localPath targetUID:targetUID targetGID:targetGID bytesWritten:&written error:&myError];
if (ret) {
HSLogDebug(@"wrote %qu bytes to %@", written, localPath);
} else {
if (error != NULL) {
*error = myError;
}
HSLogError(@"error making pack %@ local at %@: %@", packId, localPath, [myError localizedDescription]);
}
return ret;
}
- (PackIndexEntry *)doEntryForSHA1:(NSString *)sha1 error:(NSError **)error {
NSData *sha1Hex = [sha1 hexStringToData:error];
if (sha1Hex == nil) {
return nil;
}
unsigned char *sha1Bytes = (unsigned char *)[sha1Hex bytes];
HSLogTrace(@"looking for sha1 %@ in packindex %@", sha1, packId);
int fd = open([localPath fileSystemRepresentation], O_RDONLY);
if (fd == -1) {
int errnum = errno;
HSLogError(@"open(%@) error %d: %s", localPath, errnum, strerror(errnum));
SETNSERROR(@"UnixErrorDomain", errnum, @"failed to open %@: %s", localPath, strerror(errnum));
return nil;
}
uint32_t startIndex;
uint32_t endIndex;
if (![self readFanoutStartIndex:&startIndex fanoutEndIndex:&endIndex fromFD:fd forSHA1FirstByte:(unsigned int)sha1Bytes[0] error:error]) {
close(fd);
return nil;
}
close(fd);
if (endIndex == 0) {
SETNSERROR(@"PacksErrorDomain", ERROR_NOT_FOUND, @"sha1 %@ not found in pack", sha1);
return NO;
}
fd = open([localPath fileSystemRepresentation], O_RDONLY);
if (fd == -1) {
int errnum = errno;
HSLogError(@"open(%@) error %d: %s", localPath, errnum, strerror(errnum));
SETNSERROR(@"UnixErrorDomain", errnum, @"failed to open %@: %s", localPath, strerror(errnum));
return nil;
}
PackIndexEntry *ret = [self findEntryForSHA1:sha1 fd:fd betweenStartIndex:startIndex andEndIndex:endIndex error:error];
close(fd);
if (ret != nil) {
HSLogTrace(@"found sha1 %@ in packindex %@", sha1, packId);
}
return ret;
}
- (PackIndexEntry *)findEntryForSHA1:(NSString *)sha1 fd:(int)fd betweenStartIndex:(uint32_t)startIndex andEndIndex:(uint32_t)endIndex error:(NSError **)error {
NSData *sha1Data = [sha1 hexStringToData:error];
if (sha1Data == nil) {
return nil;
}
const void *sha1Bytes = [sha1Data bytes];
uint32_t lengthToMap = 4 + 4 + 256*4 + endIndex * sizeof(index_object);
pack_index *the_pack_index = mmap(0, lengthToMap, PROT_READ, MAP_SHARED, fd, 0);
if (the_pack_index == MAP_FAILED) {
int errnum = errno;
HSLogError(@"mmap(%@) error %d: %s", localPath, errnum, strerror(errnum));
SETNSERROR(@"UnixErrorDomain", errnum, @"error mapping %@ to memory: %s", localPath, strerror(errnum));
return NO;
}
int64_t left = startIndex;
int64_t right = endIndex - 1;
int64_t middle;
int64_t offset;
int64_t dataLength;
PackIndexEntry *pie = nil;
while (left <= right) {
middle = (left + right)/2;
index_object *firstIndexObject = &(the_pack_index->first_index_object);
index_object *middleIndexObject = &firstIndexObject[middle];
void *middleSHA1 = middleIndexObject->sha1;
NSComparisonResult result = [BinarySHA1 compare:middleSHA1 to:sha1Bytes];
switch (result) {
case NSOrderedAscending:
left = middle + 1;
break;
case NSOrderedDescending:
right = middle - 1;
break;
default:
offset = OSSwapBigToHostInt64(middleIndexObject->nbo_offset);
dataLength = OSSwapBigToHostInt64(middleIndexObject->nbo_datalength);
pie = [[[PackIndexEntry alloc] initWithPackId:packId offset:offset dataLength:dataLength objectSHA1:sha1] autorelease];
}
if (pie != nil) {
break;
}
}
if (munmap(the_pack_index, lengthToMap) == -1) {
int errnum = errno;
HSLogError(@"munmap: %s", strerror(errnum));
}
if (pie == nil) {
SETNSERROR(@"PackErrorDomain", ERROR_NOT_FOUND, @"sha1 %@ not found in pack %@", sha1, packId);
}
return pie;
}
- (BOOL)readFanoutStartIndex:(uint32_t *)start fanoutEndIndex:(uint32_t *)end fromFD:(int)fd forSHA1FirstByte:(unsigned int)firstByte error:(NSError **)error {
size_t len = 4 + 4 + 4*256;
uint32_t *map = mmap(0, len, PROT_READ, MAP_SHARED, fd, 0);
if (map == MAP_FAILED) {
int errnum = errno;
HSLogError(@"mmap(%@) error %d: %s", localPath, errnum, strerror(errnum));
SETNSERROR(@"UnixErrorDomain", errnum, @"error mapping %@ to memory: %s", localPath, strerror(errnum));
return NO;
}
BOOL ret = YES;
uint32_t magicNumber = OSSwapBigToHostInt32(map[0]);
uint32_t version = OSSwapBigToHostInt32(map[1]);
if (magicNumber != 0xff744f63 || version != 2) {
SETNSERROR(@"PackErrorDomain", -1, @"invalid pack index header");
ret = NO;
} else {
uint32_t *fanoutTable = map + 2;
*start = 0;
if (firstByte > 0) {
*start = OSSwapBigToHostInt32(fanoutTable[firstByte - 1]);
}
*end = OSSwapBigToHostInt32(fanoutTable[firstByte]);
}
if (munmap(map, len) == -1) {
int errnum = errno;
HSLogError(@"munmap: %s", strerror(errnum));
}
return ret;
}
@end

47
glacierrestore/GlacierPackSet.h Normal file
View file

@ -0,0 +1,47 @@
//
// GlacierPackSet.h
//
// Created by Stefan Reitshamer on 11/3/12.
// Copyright (c) 2012 Stefan Reitshamer. All rights reserved.
//
@class S3Service;
@class GlacierService;
@class PackIndexEntry;
@class GlacierPackIndex;
@class Target;
@protocol TargetConnectionDelegate;
@interface GlacierPackSet : NSObject {
Target *target;
S3Service *s3;
GlacierService *glacier;
NSString *s3BucketName;
NSString *computerUUID;
NSString *packSetName;
uid_t targetUID;
uid_t targetGID;
BOOL loadedPIEs;
NSMutableDictionary *glacierPackIndexesByPackSHA1;
NSMutableDictionary *packIndexEntriesByObjectSHA1;
}
+ (NSString *)errorDomain;
+ (unsigned long long)maxPackFileSizeMB;
+ (unsigned long long)maxPackItemSizeBytes;
- (id)initWithTarget:(Target *)theTarget
s3:(S3Service *)theS3
glacier:(GlacierService *)theGlacier
vaultName:(NSString *)theVaultName
s3BucketName:(NSString *)theS3BucketName
computerUUID:(NSString *)theComputerUUID
packSetName:(NSString *)thePackSetName
targetUID:(uid_t)theTargetUID
targetGID:(uid_t)theTargetGID;
- (BOOL)containsBlob:(BOOL *)contains forSHA1:(NSString *)sha1 dataSize:(unsigned long long *)dataSize targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error;
- (GlacierPackIndex *)glacierPackIndexForObjectSHA1:(NSString *)theObjectSHA1 targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error;
- (PackIndexEntry *)packIndexEntryForObjectSHA1:(NSString *)theSHA1 targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error;
@end

134
glacierrestore/GlacierPackSet.m Normal file
View file

@ -0,0 +1,134 @@
//
// GlacierPackSet.m
//
// Created by Stefan Reitshamer on 11/3/12.
// Copyright (c) 2012 Stefan Reitshamer. All rights reserved.
//
#import "GlacierPackSet.h"
#import "PackIndexEntry.h"
#import "GlacierPackIndex.h"
#import "PackId.h"
static unsigned long long DEFAULT_MAX_PACK_FILE_SIZE_MB = 5;
static unsigned long long DEFAULT_MAX_PACK_ITEM_SIZE_BYTES = 65536;
@implementation GlacierPackSet
+ (NSString *)errorDomain {
return @"GlacierPackSetErrorDomain";
}
+ (unsigned long long)maxPackFileSizeMB {
return DEFAULT_MAX_PACK_FILE_SIZE_MB;
}
+ (unsigned long long)maxPackItemSizeBytes {
return DEFAULT_MAX_PACK_ITEM_SIZE_BYTES;
}
- (id)initWithTarget:(Target *)theTarget
s3:(S3Service *)theS3
glacier:(GlacierService *)theGlacier
vaultName:(NSString *)theVaultName
s3BucketName:(NSString *)theS3BucketName
computerUUID:(NSString *)theComputerUUID
packSetName:(NSString *)thePackSetName
targetUID:(uid_t)theTargetUID
targetGID:(uid_t)theTargetGID {
if (self = [super init]) {
target = [theTarget retain];
s3 = [theS3 retain];
glacier = [theGlacier retain];
s3BucketName = [theS3BucketName retain];
computerUUID = [theComputerUUID retain];
packSetName = [thePackSetName retain];
targetUID = theTargetUID;
targetGID = theTargetGID;
glacierPackIndexesByPackSHA1 = [[NSMutableDictionary alloc] init];
packIndexEntriesByObjectSHA1 = [[NSMutableDictionary alloc] init];
}
return self;
}
- (void)dealloc {
[target release];
[s3 release];
[glacier release];
[s3BucketName release];
[computerUUID release];
[packSetName release];
[glacierPackIndexesByPackSHA1 release];
[packIndexEntriesByObjectSHA1 release];
[super dealloc];
}
- (BOOL)containsBlob:(BOOL *)contains forSHA1:(NSString *)sha1 dataSize:(unsigned long long *)dataSize targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error {
if (!loadedPIEs && ![self loadPackIndexEntriesWithTargetConnectionDelegate:theTCD error:error]) {
return NO;
}
PackIndexEntry *pie = [packIndexEntriesByObjectSHA1 objectForKey:sha1];
*contains = (pie != nil);
if (pie != nil && dataSize != NULL) {
*dataSize = [pie dataLength];
}
return YES;
}
- (GlacierPackIndex *)glacierPackIndexForObjectSHA1:(NSString *)theObjectSHA1 targetConnectionDelegate:(id<TargetConnectionDelegate>)theTCD error:(NSError **)error {
PackIndexEntry *pie = [self packIndexEntryForObjectSHA1:theObjectSHA1 targetConnectionDelegate:theTCD error:error];
return [glacierPackIndexesByPackSHA1 objectForKey:[[pie packId] packSHA1]];
}
- (PackIndexEntry *)packIndexEntryForObjectSHA1:(NSString *)theSHA1 targetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error {
if (!loadedPIEs && ![self loadPackIndexEntriesWithTargetConnectionDelegate:theTCD error:error]) {
return NO;
}
PackIndexEntry *ret = [packIndexEntriesByObjectSHA1 objectForKey:theSHA1];
if (ret == nil) {
SETNSERROR([GlacierPackSet errorDomain], ERROR_NOT_FOUND, @"object %@ not found in GlacierPackSet", theSHA1);
}
return ret;
}
#pragma mark internal
- (BOOL)loadPackIndexEntriesWithTargetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD error:(NSError **)error {
NSDictionary *thePackIndexEntriesByObjectSHA1 = [self packIndexEntriesBySHA1WithTargetConnectionDelegate:theTCD :error];
if (thePackIndexEntriesByObjectSHA1 == nil) {
return NO;
}
[packIndexEntriesByObjectSHA1 setDictionary:thePackIndexEntriesByObjectSHA1];
loadedPIEs = YES;
return YES;
}
- (NSDictionary *)packIndexEntriesBySHA1WithTargetConnectionDelegate:(id <TargetConnectionDelegate>)theTCD :(NSError **)error {
NSArray *glacierPackIndexes = [GlacierPackIndex glacierPackIndexesForTarget:target s3Service:s3 s3BucketName:s3BucketName computerUUID:computerUUID packSetName:packSetName targetConnectionDelegate:theTCD targetUID:targetUID targetGID:targetGID error:error];
if (glacierPackIndexes == nil) {
return nil;
}
NSMutableDictionary *packIndexEntriesBySHA1 = [NSMutableDictionary dictionary];
for (GlacierPackIndex *index in glacierPackIndexes) {
if (![index makeLocalWithTargetConnectionDelegate:theTCD error:error]) {
return nil;
}
NSArray *pies = [index allPackIndexEntriesWithTargetConnectionDelegate:theTCD error:error];
if (pies == nil) {
return nil;
}
unsigned long long packLength = 0;
for (PackIndexEntry *pie in pies) {
[packIndexEntriesBySHA1 setObject:pie forKey:[pie objectSHA1]];
unsigned long long endOffset = [pie offset] + [pie dataLength];
if (endOffset > packLength) {
packLength = endOffset;
}
}
}
[glacierPackIndexesByPackSHA1 removeAllObjects];
for (GlacierPackIndex *index in glacierPackIndexes) {
[glacierPackIndexesByPackSHA1 setObject:index forKey:[[index packId] packSHA1]];
}
return packIndexEntriesBySHA1;
}
@end

69
glacierrestore/GlacierRestorer.h Normal file
View file

@ -0,0 +1,69 @@
//
// GlacierRestorer.h
// Arq
//
// Created by Stefan Reitshamer on 5/29/13.
// Copyright (c) 2013 Stefan Reitshamer. All rights reserved.
//
#import "Restorer.h"
#import "TargetConnection.h"
@class GlacierRestorerParamSet;
@protocol GlacierRestorerDelegate;
@class SNS;
@class SQS;
@class S3Service;
@class GlacierService;
@class GlacierPackSet;
@class Repo;
@class Commit;
@class Tree;
@class BlobKey;
@interface GlacierRestorer : NSObject <Restorer, TargetConnectionDelegate> {
GlacierRestorerParamSet *paramSet;
id <GlacierRestorerDelegate> delegate;
unsigned long long bytesToRequestPerRound;
NSDate *dateToResumeRequesting;
NSString *skipFilesRoot;
NSMutableDictionary *hardlinks;
NSString *jobUUID;
SNS *sns;
SQS *sqs;
S3Service *s3;
GlacierService *glacier;
GlacierPackSet *glacierPackSet;
NSMutableSet *requestedGlacierPackSHA1s;
NSMutableDictionary *requestedGlacierPacksByPackSHA1;
NSMutableArray *glacierPacksToDownload;
NSMutableArray *calculateItems;
NSMutableArray *glacierRequestItems;
NSMutableArray *restoreItems;
NSMutableSet *requestedArchiveIds;
NSString *topicArn;
NSURL *queueURL;
NSString *queueArn;
NSString *subscriptionArn;
Repo *repo;
Commit *commit;
NSString *commitDescription;
Tree *rootTree;
NSUInteger roundsCompleted;
unsigned long long bytesRequestedThisRound;
unsigned long long bytesRequested;
unsigned long long totalBytesToRequest;
unsigned long long bytesTransferred;
unsigned long long totalBytesToTransfer;
unsigned long long writtenToCurrentFile;
}
- (id)initWithGlacierRestorerParamSet:(GlacierRestorerParamSet *)theParamSet
delegate:(id <GlacierRestorerDelegate>)theDelegate;
- (void)run;
@end

964
glacierrestore/GlacierRestorer.m Normal file
View file

@ -0,0 +1,964 @@
//
// GlacierRestorer.m
// Arq
//
// Created by Stefan Reitshamer on 5/29/13.
// Copyright (c) 2013 Stefan Reitshamer. All rights reserved.
//
#import "GlacierRestorer.h"
#import "GlacierRestorerParamSet.h"
#import "GlacierRestorerDelegate.h"
#import "S3AuthorizationProvider.h"
#import "S3Service.h"
#import "ArqSalt.h"
#import "Repo.h"
#import "Commit.h"
#import "Tree.h"
#import "Node.h"
#import "GlacierAuthorizationProvider.h"
#import "GlacierService.h"
#import "FileOutputStream.h"
#import "NSFileManager_extra.h"
#import "BlobKey.h"
#import "NSData-GZip.h"
#import "FileAttributes.h"
#import "BufferedOutputStream.h"
#import "OSStatusDescription.h"
#import "FileACL.h"
#import "BufferedInputStream.h"
#import "DataInputStream.h"
#import "XAttrSet.h"
#import "FileInputStream.h"
#import "SHA1Hash.h"
#import "PackIndexEntry.h"
#import "UserLibrary_Arq.h"
#import "SNS.h"
#import "SQS.h"
#import "NSString_extra.h"
#import "ReceiveMessageResponse.h"
#import "SQSMessage.h"
#import "NSObject+SBJSON.h"
#import "NSString+SBJSON.h"
#import "RestoreItem.h"
#import "GlacierRequestItem.h"
#import "CalculateItem.h"
#import "Bucket.h"
#import "Target.h"
#import "GlacierPackSet.h"
#import "GlacierPack.h"
#import "GlacierPackIndex.h"
#import "AWSRegion.h"
#import "Streams.h"
#define WAIT_TIME (3.0)
#define SLEEP_CYCLES (2)
#define MAX_QUEUE_MESSAGES_TO_READ (10)
#define MAX_GLACIER_RETRIES (10)
#define RESTORE_DAYS (10)
@implementation GlacierRestorer
- (id)initWithGlacierRestorerParamSet:(GlacierRestorerParamSet *)theParamSet
delegate:(id <GlacierRestorerDelegate>)theDelegate {
if (self = [super init]) {
paramSet = [theParamSet retain];
delegate = theDelegate; // Don't retain it.
bytesToRequestPerRound = paramSet.downloadBytesPerSecond * 60 * 60 * 4; // 4 hours at preferred download rate
dateToResumeRequesting = [[NSDate date] retain];
skipFilesRoot = [[[UserLibrary arqUserLibraryPath] stringByAppendingFormat:@"/RestoreJobSkipFiles/%f", [NSDate timeIntervalSinceReferenceDate]] retain];
hardlinks = [[NSMutableDictionary alloc] init];
jobUUID = [[NSString stringWithRandomUUID] retain];
requestedGlacierPacksByPackSHA1 = [[NSMutableDictionary alloc] init];
glacierPacksToDownload = [[NSMutableArray alloc] init];
calculateItems = [[NSMutableArray alloc] init];
glacierRequestItems = [[NSMutableArray alloc] init];
restoreItems = [[NSMutableArray alloc] init];
requestedArchiveIds = [[NSMutableSet alloc] init];
}
return self;
}
- (void)dealloc {
[paramSet release];
[dateToResumeRequesting release];
[skipFilesRoot release];
[hardlinks release];
[jobUUID release];
[sns release];
[sqs release];
[s3 release];
[glacier release];
[glacierPackSet release];
[requestedGlacierPacksByPackSHA1 release];
[glacierPacksToDownload release];
[calculateItems release];
[glacierRequestItems release];
[restoreItems release];
[requestedArchiveIds release];
[topicArn release];
[queueURL release];
[queueArn release];
[subscriptionArn release];
[repo release];
[commit release];
[commitDescription release];
[rootTree release];
[super dealloc];
}
- (void)run {
HSLogDebug(@"GlacierRestorer starting");
NSError *myError = nil;
if (![self run:&myError]) {
HSLogDebug(@"[GlacierRestorer run:] failed; %@", myError);
[delegate glacierRestorerDidFail:myError];
} else {
HSLogDebug(@"[GlacierRestorer run:] succeeded");
[delegate glacierRestorerDidSucceed];
}
[self deleteTopic];
[self deleteQueue];
NSError *removeError = nil;
if ([[NSFileManager defaultManager] fileExistsAtPath:skipFilesRoot] && ![[NSFileManager defaultManager] removeItemAtPath:skipFilesRoot error:&removeError]) {
HSLogError(@"failed to remove %@: %@", skipFilesRoot, removeError);
}
HSLogDebug(@"GlacierRestorer finished");
}
- (NSNumber *)isObjectAvailableForBlobKey:(BlobKey *)theBlobKey error:(NSError **)error {
if ([theBlobKey storageType] == StorageTypeS3) {
// In Repo.m doPutData (line 503) we were incorrectly creating a BlobKey with storageType hard-coded to StorageTypeS3 when it should have been StorageTypeS3Glacier.
// Since we're here because we're restoring from a StorageTypeGlacier folder, we'll assume the storageType should be StorageTypeS3Glacier instead of StorageTypeS3.
theBlobKey = [[[BlobKey alloc] initCopyOfBlobKey:theBlobKey withStorageType:StorageTypeS3Glacier] autorelease];
}
if ([theBlobKey storageType] == StorageTypeS3Glacier) {
return [repo isObjectDownloadableForBlobKey:theBlobKey error:error];
}
// Packed blobs have sha1, but not archiveId.
if ([theBlobKey archiveId] == nil) {
return [NSNumber numberWithBool:YES];
}
NSError *myError = nil;
NSString *jobId = [self completedJobIdForArchiveId:[theBlobKey archiveId] error:&myError];
if (jobId == nil) {
if ([myError code] == ERROR_GLACIER_OBJECT_NOT_AVAILABLE) {
return [NSNumber numberWithBool:NO];
}
SETERRORFROMMYERROR;
return nil;
}
return [NSNumber numberWithBool:YES];
}
- (NSNumber *)sizeOfBlob:(BlobKey *)theBlobKey error:(NSError **)error {
if ([theBlobKey storageType] == StorageTypeGlacier) {
return [NSNumber numberWithUnsignedLongLong:[theBlobKey archiveSize]];
}
unsigned long long dataSize = 0;
NSNumber *contains = [repo containsBlobForBlobKey:theBlobKey dataSize:&dataSize error:error];
if (contains == nil) {
return NO;
}
if (![contains boolValue]) {
// We'll report this to the user as an error during the download phase.
HSLogError(@"repo does not contain %@!", theBlobKey);
}
return [NSNumber numberWithUnsignedLongLong:dataSize];
}
- (BOOL)requestBlobKey:(BlobKey *)theBlobKey error:(NSError **)error {
if (theBlobKey == nil) {
return YES;
}
if ([theBlobKey storageType] == StorageTypeS3) {
// In Repo.m doPutData (line 503) we were incorrectly creating a BlobKey with storageType hard-coded to StorageTypeS3 when it should have been StorageTypeS3Glacier.
// Since we're here because we're restoring from a StorageTypeGlacier folder, we'll assume the storageType should be StorageTypeS3Glacier instead of StorageTypeS3.
theBlobKey = [[[BlobKey alloc] initCopyOfBlobKey:theBlobKey withStorageType:StorageTypeS3Glacier] autorelease];
}
if ([theBlobKey storageType] == StorageTypeS3Glacier) {
unsigned long long dataSize = 0;
NSNumber *contains = [repo containsBlobForBlobKey:theBlobKey dataSize:&dataSize error:error];
if (contains == nil) {
return NO;
}
if (![contains boolValue]) {
// We'll report this to the user as an error during the download phase.
HSLogError(@"repo does not contain %@!", theBlobKey);
} else {
BOOL alreadyRestoredOrRestoring = NO;
if (![repo restoreObjectForBlobKey:theBlobKey forDays:RESTORE_DAYS alreadyRestoredOrRestoring:&alreadyRestoredOrRestoring error:error]) {
return NO;
}
if (![self addToBytesRequested:dataSize error:error]) {
return NO;
}
}
return YES;
}
if ([theBlobKey archiveId] != nil) {
if (![requestedArchiveIds containsObject:[theBlobKey archiveId]]) {
if (![glacier initiateRetrievalJobForVaultName:[[paramSet bucket] vaultName]
archiveId:[theBlobKey archiveId]
snsTopicArn:topicArn
error:error]) {
return NO;
}
[requestedArchiveIds addObject:[theBlobKey archiveId]];
HSLogDebug(@"requested %@", theBlobKey);
}
if (![self addToBytesRequested:[theBlobKey archiveSize] error:error]) {
return NO;
}
}
return YES;
}
#pragma mark Restorer
- (NSString *)errorDomain {
return @"GlacierRestorerErrorDomain";
}
- (NSData *)dataForBlobKey:(BlobKey *)theBlobKey error:(NSError **)error {
if ([theBlobKey storageType] == StorageTypeS3) {
// In Repo.m doPutData (line 503) we were incorrectly creating a BlobKey with storageType hard-coded to StorageTypeS3 when it should have been StorageTypeS3Glacier.
// Since we're here because we're restoring from a StorageTypeGlacier folder, we'll assume the storageType should be StorageTypeS3Glacier instead of StorageTypeS3.
theBlobKey = [[[BlobKey alloc] initCopyOfBlobKey:theBlobKey withStorageType:StorageTypeS3Glacier] autorelease];
}
if ([theBlobKey storageType] == StorageTypeS3Glacier) {
NSData *data = [repo dataForBlobKey:theBlobKey error:error];
if (data == nil) {
return nil;
}
if (![self addToBytesTransferred:(unsigned long long)[data length] error:error]) {
return nil;
}
return data;
}
NSData *ret = nil;
if ([theBlobKey archiveId] == nil) {
// Packed blob.
PackIndexEntry *pie = [glacierPackSet packIndexEntryForObjectSHA1:[theBlobKey sha1] targetConnectionDelegate:self error:error];
if (pie == nil) {
return nil;
}
GlacierPack *glacierPack = [requestedGlacierPacksByPackSHA1 objectForKey:[[pie packId] packSHA1]];
if (glacierPack == nil) {
SETNSERROR([self errorDomain], -1, @"no GlacierPack for packSHA1 %@", [[pie packId] packSHA1]);
return nil;
}
ret = [glacierPack cachedDataForObjectAtOffset:[pie offset] error:error];
} else {
NSString *completedJobId = [self completedJobIdForArchiveId:[theBlobKey archiveId] error:error];
if (completedJobId == nil) {
return nil;
}
ret = [glacier dataForVaultName:[[paramSet bucket] vaultName] jobId:completedJobId retries:MAX_GLACIER_RETRIES error:error];
if (ret != nil) {
if (![self addToBytesTransferred:[ret length] error:error]) {
return nil;
}
}
}
if (ret != nil) {
ret = [repo decryptData:ret error:error];
}
return ret;
}
- (BOOL)shouldSkipFile:(NSString *)thePath {
NSString *skipFilePath = [skipFilesRoot stringByAppendingString:thePath];
return [[NSFileManager defaultManager] fileExistsAtPath:skipFilePath];
}
- (BOOL)useTargetUIDAndGID {
return paramSet.useTargetUIDAndGID;
}
- (uid_t)targetUID {
return paramSet.targetUID;
}
- (gid_t)targetGID {
return paramSet.targetGID;
}
#pragma mark TargetConnectionDelegate
- (BOOL)targetConnectionShouldRetryOnTransientError:(NSError **)error {
return YES;
}
#pragma mark internal
- (BOOL)run:(NSError **)error {
if (![self setUp:error]) {
return NO;
}
NSString *calculatingMessage = @"Calculating sizes";
if ([[NSFileManager defaultManager] fileExistsAtPath:paramSet.destinationPath]) {
calculatingMessage = @"Comparing existing files to backup data";
}
if ([delegate glacierRestorerMessageDidChange:calculatingMessage]) {
SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested");
return NO;
}
if (![self calculateSizes:error]) {
return NO;
}
if ([delegate glacierRestorerMessageDidChange:[NSString stringWithFormat:@"Restoring %@ from %@ to %@", paramSet.rootItemName, commitDescription, paramSet.destinationPath]]) {
SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested");
return NO;
}
if (![self findNeededGlacierPacks:error]) {
return NO;
}
// Just request all the Glacier packs right away. It probably won't amount to more than 4 hours' worth of downloads.
for (GlacierPack *glacierPack in [requestedGlacierPacksByPackSHA1 allValues]) {
if (![glacier initiateRetrievalJobForVaultName:[[paramSet bucket] vaultName]
archiveId:[glacierPack archiveId]
snsTopicArn:topicArn
error:error]) {
return NO;
}
if (![self addToBytesRequested:[glacierPack packSize] error:error]) {
return NO;
}
}
BOOL restoredAnItem = NO;
BOOL ret = YES;
NSAutoreleasePool *pool = nil;
for (;;) {
[pool drain];
pool = [[NSAutoreleasePool alloc] init];
// Reset counters if necessary.
if (bytesRequestedThisRound >= bytesToRequestPerRound) {
roundsCompleted++;
bytesRequestedThisRound = 0;
NSDate *nextResumeDate = [[dateToResumeRequesting dateByAddingTimeInterval:(60 * 60 * 4)] retain];
[dateToResumeRequesting release];
dateToResumeRequesting = nextResumeDate;
HSLogDebug(@"reset next request resume date to %@", nextResumeDate);
}
// Make sure we've transferred all the bytes from all but the most recent round of requests.
double theMinimum = (roundsCompleted = 0) ? 0 : ((double)bytesToRequestPerRound * (double)(roundsCompleted - 1)) * .9;
unsigned long long minimumBytesToHaveTransferred = (unsigned long long)theMinimum;
if ((bytesRequestedThisRound < bytesToRequestPerRound)
&& (bytesTransferred >= minimumBytesToHaveTransferred)
&& ([[NSDate date] earlierDate:dateToResumeRequesting] == dateToResumeRequesting)) {
// Request more Glacier items.
if (![self requestMoreGlacierItems:error]) {
ret = NO;
break;
}
}
if (!restoredAnItem) {
// Read any available items from the queue.
HSLogDebug(@"reading queue");
if (![self readQueue:error]) {
ret = NO;
break;
}
}
if ([glacierPacksToDownload count] == 0 && [restoreItems count] == 0) {
HSLogDebug(@"finished requesting");
if ([delegate glacierRestorerDidFinishRequesting]) {
SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested");
ret = NO;
break;
}
break;
}
// Restore an item if possible.
if ([glacierPacksToDownload count] > 0) {
GlacierPack *glacierPack = [glacierPacksToDownload objectAtIndex:0];
NSError *myError = nil;
NSString *completedJobId = [self completedJobIdForArchiveId:[glacierPack archiveId] error:&myError];
if (completedJobId == nil) {
if ([myError code] != ERROR_GLACIER_OBJECT_NOT_AVAILABLE) {
SETERRORFROMMYERROR;
ret = NO;
break;
}
HSLogDebug(@"%@ not available yet", glacierPack);
restoredAnItem = NO;
} else {
HSLogDebug(@"downloading %@", glacierPack);
NSData *packData = [glacier dataForVaultName:[[paramSet bucket] vaultName] jobId:completedJobId retries:MAX_GLACIER_RETRIES error:error];
if (packData == nil) {
ret = NO;
break;
}
restoredAnItem = YES;
if (![self addToBytesTransferred:[packData length] error:error]) {
ret = NO;
break;
}
if (![glacierPack cachePackDataToDisk:packData error:error]) {
ret = NO;
break;
}
HSLogDebug(@"downloaded %@", glacierPack);
[glacierPacksToDownload removeObject:glacierPack];
restoredAnItem = YES;
}
} else {
NSError *restoreError = nil;
RestoreItem *restoreItem = [restoreItems objectAtIndex:0];
restoredAnItem = YES;
HSLogDebug(@"attempting to restore %@", restoreItem);
if (![restoreItem restoreWithHardlinks:hardlinks restorer:self error:&restoreError]) {
if ([restoreError code] == ERROR_GLACIER_OBJECT_NOT_AVAILABLE) {
HSLogDebug(@"glacier object not available yet");
restoredAnItem = NO;
} else if ([restoreError isErrorWithDomain:[self errorDomain] code:ERROR_ABORT_REQUESTED]) {
if (error != NULL) {
*error = restoreError;
}
ret = NO;
break;
} else {
[delegate glacierRestorerErrorMessage:[restoreError localizedDescription] didOccurForPath:[restoreItem path]];
}
}
if (restoredAnItem) {
NSArray *nextItems = [restoreItem nextItemsWithRepo:repo error:error];
if (nextItems == nil) {
ret = NO;
break;
}
[restoreItems removeObjectAtIndex:0];
if ([nextItems count] > 0) {
[restoreItems insertObjects:nextItems atIndexes:[NSIndexSet indexSetWithIndexesInRange:NSMakeRange(0, [nextItems count])]];
}
}
}
if (!restoredAnItem) {
HSLogDebug(@"sleeping");
for (NSUInteger i = 0; i < SLEEP_CYCLES; i++) {
if (![self addToBytesTransferred:0 error:error]) {
ret = NO;
break;
}
[NSThread sleepForTimeInterval:3.0];
}
}
if (!ret) {
break;
}
}
if (!ret && error != NULL) {
[*error retain];
}
[pool drain];
if (!ret && error != NULL) {
[*error autorelease];
}
return ret;
}
- (BOOL)setUp:(NSError **)error {
NSString *secretAccessKey = [[[paramSet bucket] target] secret:error];
if (secretAccessKey == nil) {
return NO;
}
AWSRegion *awsRegion = [AWSRegion regionWithS3Endpoint:[[[paramSet bucket] target] endpoint]];
if (awsRegion == nil) {
SETNSERROR([self errorDomain], -1, @"unknown AWS region %@", [[[paramSet bucket] target] endpoint]);
return NO;
}
sns = [[SNS alloc] initWithAccessKey:[[[[paramSet bucket] target] endpoint] user] secretKey:secretAccessKey awsRegion:awsRegion retryOnTransientError:YES];
sqs = [[SQS alloc] initWithAccessKey:[[[[paramSet bucket] target] endpoint] user] secretKey:secretAccessKey awsRegion:awsRegion retryOnTransientError:YES];
s3 = [[[[paramSet bucket] target] s3:error] retain];
if (s3 == nil) {
return NO;
}
GlacierAuthorizationProvider *gap = [[[GlacierAuthorizationProvider alloc] initWithAccessKey:[[[[paramSet bucket] target] endpoint] user] secretKey:secretAccessKey] autorelease];
glacier = [[GlacierService alloc] initWithGlacierAuthorizationProvider:gap awsRegion:awsRegion useSSL:YES retryOnTransientError:YES];
glacierPackSet = [[GlacierPackSet alloc] initWithTarget:[[paramSet bucket] target]
s3:s3
glacier:glacier
vaultName:[[paramSet bucket] vaultName]
s3BucketName:[[[[[paramSet bucket] target] endpoint] path] lastPathComponent]
computerUUID:[[paramSet bucket] computerUUID]
packSetName:[[[paramSet bucket] bucketUUID] stringByAppendingString:@"-glacierblobs"]
targetUID:paramSet.targetUID
targetGID:paramSet.targetGID];
ArqSalt *arqSalt = [[[ArqSalt alloc] initWithTarget:[[paramSet bucket] target] targetUID:[paramSet targetUID] targetGID:[paramSet targetGID] computerUUID:[[paramSet bucket] computerUUID]] autorelease];
NSData *salt = [arqSalt saltWithTargetConnectionDelegate:self error:error];
if (salt == nil) {
return NO;
}
repo = [[Repo alloc] initWithBucket:[paramSet bucket]
encryptionPassword:[paramSet encryptionPassword]
targetUID:[paramSet targetUID]
targetGID:[paramSet targetGID]
loadExistingMutablePackFiles:NO
targetConnectionDelegate:self
repoDelegate:nil
error:error];
if (repo == nil) {
return NO;
}
commit = [[repo commitForBlobKey:paramSet.commitBlobKey error:error] retain];
if (commit == nil) {
return NO;
}
NSDateFormatter *dateFormatter = [[[NSDateFormatter alloc] init] autorelease];
[dateFormatter setDateStyle:NSDateFormatterMediumStyle];
[dateFormatter setTimeStyle:NSDateFormatterShortStyle];
commitDescription = [[dateFormatter stringFromDate:[commit creationDate]] retain];
rootTree = [[repo treeForBlobKey:paramSet.treeBlobKey error:error] retain];
if (rootTree == nil) {
return NO;
}
if ([delegate glacierRestorerMessageDidChange:[NSString stringWithFormat:@"Creating SNS topic and SQS queue"]]) {
SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested");
return NO;
}
topicArn = [[sns createTopic:[NSString stringWithFormat:@"%@_topic", jobUUID] error:error] retain];
if (topicArn == nil) {
return NO;
}
queueURL = [[sqs createQueueWithName:[NSString stringWithFormat:@"%@_queue", jobUUID] error:error] retain];
if (queueURL == nil) {
return NO;
}
queueArn = [[sqs queueArnForQueueURL:queueURL error:error] retain];
if (queueArn == nil) {
return NO;
}
if (![sqs setSendMessagePermissionToQueueURL:queueURL queueArn:queueArn forSourceArn:topicArn error:error]) {
return NO;
}
subscriptionArn = [[sns subscribeQueueArn:queueArn toTopicArn:topicArn error:error] retain];
if (subscriptionArn == nil) {
return NO;
}
if (paramSet.nodeName != nil) {
Node *node = [rootTree childNodeWithName:paramSet.nodeName];
if ([[rootTree childNodeNames] isEqualToArray:[NSArray arrayWithObject:@"."]]) {
// The single-file case.
node = [rootTree childNodeWithName:@"."];
}
NSAssert(node != nil, @"node can't be nil");
[calculateItems addObject:[[[CalculateItem alloc] initWithPath:paramSet.destinationPath node:node] autorelease]];
[glacierRequestItems addObject:[[[GlacierRequestItem alloc] initWithPath:paramSet.destinationPath node:node] autorelease]];
[restoreItems addObject:[[[RestoreItem alloc] initWithPath:paramSet.destinationPath tree:rootTree node:node] autorelease]];
} else {
[calculateItems addObject:[[[CalculateItem alloc] initWithPath:paramSet.destinationPath tree:rootTree] autorelease]];
[glacierRequestItems addObject:[[[GlacierRequestItem alloc] initWithPath:paramSet.destinationPath tree:rootTree] autorelease]];
[restoreItems addObject:[[[RestoreItem alloc] initWithPath:paramSet.destinationPath tree:rootTree] autorelease]];
}
return YES;
}
- (BOOL)calculateSizes:(NSError **)error {
BOOL ret = YES;
NSAutoreleasePool *pool = nil;
while ([calculateItems count] > 0) {
[pool drain];
pool = [[NSAutoreleasePool alloc] init];
CalculateItem *item = [calculateItems objectAtIndex:0];
if (![item calculateWithRepo:repo restorer:self error:error]) {
ret = NO;
break;
}
for (NSString *path in [item filesToSkip]) {
[self skipFile:path];
}
unsigned long long bytesToTransfer = [item bytesToTransfer];
if (![self addToTotalBytesToRequest:bytesToTransfer error:error]) {
ret = NO;
break;
}
if (![self addToTotalBytesToTransfer:bytesToTransfer error:error]) {
ret = NO;
break;
}
[calculateItems removeObjectAtIndex:0];
NSArray *nextItems = [item nextItems];
if ([nextItems count] > 0) {
[calculateItems insertObjects:nextItems atIndexes:[NSIndexSet indexSetWithIndexesInRange:NSMakeRange(0, [nextItems count])]];
}
}
if (!ret && error != NULL) {
[*error retain];
}
[pool drain];
if (!ret && error != NULL) {
[*error autorelease];
}
return ret;
}
- (void)skipFile:(NSString *)thePath {
NSString *skipFilePath = [skipFilesRoot stringByAppendingString:thePath];
NSError *myError = nil;
if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:skipFilePath targetUID:paramSet.targetUID targetGID:paramSet.targetGID error:&myError]) {
HSLogError(@"error creating parent dir for %@: %@", skipFilePath, myError);
return;
}
if (![[NSFileManager defaultManager] touchFileAtPath:skipFilePath targetUID:paramSet.targetUID targetGID:paramSet.targetGID error:&myError]) {
HSLogError(@"error touching %@: %@", skipFilePath, myError);
}
HSLogDebug(@"skip file %@", thePath);
}
- (BOOL)findNeededGlacierPacks:(NSError **)error {
HSLogDebug(@"finding needed glacier packs");
if (paramSet.nodeName != nil) {
Node *node = [rootTree childNodeWithName:paramSet.nodeName];
if ([[rootTree childNodeNames] isEqualToArray:[NSArray arrayWithObject:@"."]]) {
// The single-file case.
node = [rootTree childNodeWithName:@"."];
}
NSAssert(node != nil, @"node can't be nil");
if (![self findNeededGlacierPacksForNode:node path:paramSet.destinationPath error:error]) {
return NO;
}
} else {
if (![self findNeededGlacierPacksForTree:rootTree path:paramSet.destinationPath error:error]) {
return NO;
}
}
[glacierPacksToDownload setArray:[requestedGlacierPacksByPackSHA1 allValues]];
HSLogDebug(@"found %ld needed glacier packs", (unsigned long)[requestedGlacierPacksByPackSHA1 count]);
return YES;
}
- (BOOL)findNeededGlacierPacksForTree:(Tree *)theTree path:(NSString *)thePath error:(NSError **)error {
HSLogDebug(@"requesting glacier packs for tree xattrs %@", [theTree xattrsBlobKey]);
if (![self findNeededGlacierPackForBlobKey:[theTree xattrsBlobKey] error:error]) {
return NO;
}
HSLogDebug(@"requesting glacier packs for tree acl %@", [theTree aclBlobKey]);
if (![self findNeededGlacierPackForBlobKey:[theTree aclBlobKey] error:error]) {
return NO;
}
BOOL ret = YES;
NSAutoreleasePool *pool = nil;
for (NSString *nodeName in [theTree childNodeNames]) {
[pool drain];
pool = [[NSAutoreleasePool alloc] init];
Node *node = [theTree childNodeWithName:nodeName];
NSString *childPath = [thePath stringByAppendingPathComponent:nodeName];
if ([node isTree]) {
Tree *childTree = [repo treeForBlobKey:[node treeBlobKey] error:error];
if (childTree == nil) {
ret = NO;
break;
}
if (![self findNeededGlacierPacksForTree:childTree path:childPath error:error]) {
ret = NO;
break;
}
} else {
if (![self findNeededGlacierPacksForNode:node path:childPath error:error]) {
ret = NO;
break;
}
}
}
if (!ret && error != NULL) {
[*error retain];
}
[pool drain];
if (!ret && error != NULL) {
[*error autorelease];
}
return ret;
}
- (BOOL)findNeededGlacierPacksForNode:(Node *)theNode path:(NSString *)thePath error:(NSError **)error {
if (![self findNeededGlacierPackForBlobKey:[theNode xattrsBlobKey] error:error]) {
return NO;
}
if (![self findNeededGlacierPackForBlobKey:[theNode aclBlobKey] error:error]) {
return NO;
}
BOOL ret = YES;
if (![self shouldSkipFile:thePath]) {
for (BlobKey *dataBlobKey in [theNode dataBlobKeys]) {
if (![self findNeededGlacierPackForBlobKey:dataBlobKey error:error]) {
ret = NO;
break;
}
}
}
return ret;
}
- (BOOL)findNeededGlacierPackForBlobKey:(BlobKey *)theBlobKey error:(NSError **)error {
// Packed blobs have sha1, but not archiveId.
if (theBlobKey != nil && [theBlobKey storageType] == StorageTypeGlacier && [theBlobKey archiveId] == nil) {
NSString *theSHA1 = [theBlobKey sha1];
NSError *myError = nil;
GlacierPackIndex *glacierPackIndex = [glacierPackSet glacierPackIndexForObjectSHA1:theSHA1 targetConnectionDelegate:self error:&myError];
if (glacierPackIndex == nil) {
if ([myError code] == ERROR_NOT_FOUND) {
HSLogError(@"object SHA1 %@ not found in any glacier pack index", theSHA1);
} else {
SETERRORFROMMYERROR;
return NO;
}
} else if (![[requestedGlacierPacksByPackSHA1 allKeys] containsObject:[[glacierPackIndex packId] packSHA1]]) {
NSString *archiveId = [glacierPackIndex archiveId:error];
if (archiveId == nil) {
return NO;
}
unsigned long long packSize = [glacierPackIndex packSize:error];
if (packSize == 0) {
return NO;
}
HSLogDebug(@"need glacier pack SHA1 %@ archiveId %@", [[glacierPackIndex packId] packSHA1], archiveId);
if (![self addToTotalBytesToRequest:packSize error:error]) {
return NO;
}
if (![self addToTotalBytesToTransfer:packSize error:error]) {
return NO;
}
GlacierPack *glacierPack = [[[GlacierPack alloc] initWithTarget:[[paramSet bucket] target]
s3BucketName:[[[[[paramSet bucket] target] endpoint] path] lastPathComponent]
computerUUID:[[paramSet bucket] computerUUID]
bucketUUID:[[paramSet bucket] bucketUUID]
packSHA1:[[glacierPackIndex packId] packSHA1]
archiveId:archiveId
packSize:packSize
targetUID:[paramSet targetUID]
targetGID:[paramSet targetGID]] autorelease];
[requestedGlacierPacksByPackSHA1 setObject:glacierPack forKey:[[glacierPackIndex packId] packSHA1]];
}
}
return YES;
}
- (BOOL)readQueue:(NSError **)error {
BOOL ret = YES;
NSAutoreleasePool *pool = nil;
for (;;) {
[pool drain];
pool = [[NSAutoreleasePool alloc] init];
ReceiveMessageResponse *response = [sqs receiveMessagesForQueueURL:queueURL maxMessages:MAX_QUEUE_MESSAGES_TO_READ error:error];
if (response == nil) {
ret = NO;
break;
}
HSLogDebug(@"got %lu messages from queue", (unsigned long)[[response messages] count]);
if ([[response messages] count] == 0) {
break;
}
for (SQSMessage *msg in [response messages]) {
if (![self processMessage:msg error:error]) {
ret = NO;
break;
}
}
}
if (!ret && error != NULL) {
[*error retain];
}
[pool drain];
if (!ret && error != NULL) {
[*error autorelease];
}
return ret;
}
- (BOOL)processMessage:(SQSMessage *)theMessage error:(NSError **)error {
NSDictionary *json = [[theMessage body] JSONValue:error];
if (json == nil) {
return NO;
}
id msgJson = [json objectForKey:@"Message"];
// Sometimes it comes back as an NSString, sometimes already as an NSDictionary?!
if ([msgJson isKindOfClass:[NSString class]]) {
msgJson = [(NSString *)msgJson JSONValue:error];
if (msgJson == nil) {
return NO;
}
}
NSDictionary *msgDict = (NSDictionary *)msgJson;
NSString *archiveId = [msgDict objectForKey:@"ArchiveId"];
NSString *jobId = [msgDict objectForKey:@"JobId"];
NSNumber *completed = [msgDict objectForKey:@"Completed"];
NSAssert([completed boolValue], @"Completed must be YES");
HSLogDetail(@"archiveId %@ is now available", archiveId);
if (![self saveCompletedJobWithJobId:jobId archiveId:archiveId error:error]) {
return NO;
}
NSError *myError = nil;
if (![sqs deleteMessageWithQueueURL:queueURL receiptHandle:[theMessage receiptHandle] error:&myError]) {
HSLogError(@"error deleting message %@ from queue %@: %@", [theMessage receiptHandle], queueURL, myError);
}
return YES;
}
- (BOOL)requestMoreGlacierItems:(NSError **)error {
BOOL ret = YES;
NSAutoreleasePool *pool = nil;
while (bytesRequestedThisRound < bytesToRequestPerRound && [glacierRequestItems count] > 0) {
[pool drain];
pool = [[NSAutoreleasePool alloc] init];
GlacierRequestItem *item = [glacierRequestItems objectAtIndex:0];
NSArray *nextItems = [item requestWithRestorer:self repo:repo error:error];
if (nextItems == nil) {
ret = NO;
break;
}
[glacierRequestItems removeObjectAtIndex:0];
if ([nextItems count] > 0) {
[glacierRequestItems insertObjects:nextItems atIndexes:[NSIndexSet indexSetWithIndexesInRange:NSMakeRange(0, [nextItems count])]];
}
}
if (!ret && error != NULL) {
[*error retain];
}
[pool drain];
if (!ret && error != NULL) {
[*error autorelease];
}
return ret;
}
- (NSString *)completedJobIdForArchiveId:(NSString *)theArchiveId error:(NSError **)error {
NSString *ret = nil;
NSString *statusPath = [self statusPathForArchiveId:theArchiveId];
if ([[NSFileManager defaultManager] fileExistsAtPath:statusPath]) {
NSDictionary *attribs = [[NSFileManager defaultManager] attributesOfItemAtPath:statusPath error:error];
if (attribs == nil) {
return NO;
}
if ([[attribs objectForKey:NSFileSize] unsignedLongLongValue] > 0) {
NSData *jobIdData = [NSData dataWithContentsOfFile:statusPath options:NSUncachedRead error:error];
if (jobIdData == nil) {
return NO;
}
ret = [[[NSString alloc] initWithData:jobIdData encoding:NSUTF8StringEncoding] autorelease];
}
}
if (!ret) {
SETNSERROR([self errorDomain], ERROR_GLACIER_OBJECT_NOT_AVAILABLE, @"object not available for archive %@", theArchiveId);
}
return ret;
}
- (BOOL)didRequestArchiveId:(NSString *)theArchiveId error:(NSError **)error {
NSString *path = [self statusPathForArchiveId:theArchiveId];
NSData *emptyData = [NSData data];
if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:path targetUID:paramSet.targetUID targetGID:paramSet.targetGID error:error]
|| ![Streams writeData:emptyData atomicallyToFile:path targetUID:[paramSet targetUID] targetGID:[paramSet targetGID] bytesWritten:NULL error:error]) {
return NO;
}
return YES;
}
- (BOOL)saveCompletedJobWithJobId:(NSString *)theJobId archiveId:(NSString *)theArchiveId error:(NSError **)error {
NSString *path = [self statusPathForArchiveId:theArchiveId];
NSData *jobIdData = [theJobId dataUsingEncoding:NSUTF8StringEncoding];
if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:path targetUID:paramSet.targetUID targetGID:paramSet.targetGID error:error]
|| ![Streams writeData:jobIdData atomicallyToFile:path targetUID:paramSet.targetUID targetGID:paramSet.targetGID bytesWritten:NULL error:error]) {
return NO;
}
return YES;
}
- (NSString *)statusPathForArchiveId:(NSString *)theArchiveId {
return [[UserLibrary arqUserLibraryPath] stringByAppendingFormat:@"/RestoreJobData/%@/%@/%@/%@",
[[paramSet bucket] computerUUID],
jobUUID,
[theArchiveId substringToIndex:2],
[theArchiveId substringFromIndex:2]];
}
- (BOOL)addToBytesRequested:(unsigned long long)length error:(NSError **)error {
bytesRequested += length;
bytesRequestedThisRound += length;
if ([delegate glacierRestorerBytesRequestedDidChange:[NSNumber numberWithUnsignedLongLong:bytesRequested]]) {
SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested");
return NO;
}
return YES;
}
- (BOOL)addToTotalBytesToRequest:(unsigned long long)length error:(NSError **)error {
totalBytesToRequest += length;
if ([delegate glacierRestorerTotalBytesToRequestDidChange:[NSNumber numberWithUnsignedLongLong:totalBytesToRequest]]) {
SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested");
return NO;
}
return YES;
}
- (BOOL)addToBytesTransferred:(unsigned long long)length error:(NSError **)error {
bytesTransferred += length;
if ([delegate glacierRestorerBytesTransferredDidChange:[NSNumber numberWithUnsignedLongLong:bytesTransferred]]) {
SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested");
return NO;
}
return YES;
}
- (BOOL)addToTotalBytesToTransfer:(unsigned long long)length error:(NSError **)error {
totalBytesToTransfer += length;
if ([delegate glacierRestorerTotalBytesToTransferDidChange:[NSNumber numberWithUnsignedLongLong:totalBytesToTransfer]]) {
SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested");
return NO;
}
return YES;
}
- (void)deleteQueue {
if (queueURL != nil) {
NSError *myError = nil;
HSLogDetail(@"delete SQS queue %@", queueURL);
if (![sqs deleteQueue:queueURL error:&myError]) {
HSLogError(@"error deleting queue %@: %@", queueURL, myError);
}
}
}
- (void)deleteTopic {
if (topicArn != nil) {
NSError *myError = nil;
HSLogDetail(@"deleting SNS topic %@", topicArn);
if (![sns deleteTopicWithArn:topicArn error:&myError]) {
HSLogError(@"error deleting SNS topic %@: %@", topicArn, myError);
}
}
}
@end

27
glacierrestore/GlacierRestorerDelegate.h Normal file
View file

@ -0,0 +1,27 @@
//
// GlacierRestorerDelegate.h
// Arq
//
// Created by Stefan Reitshamer on 5/29/13.
// Copyright (c) 2013 Stefan Reitshamer. All rights reserved.
//
@protocol GlacierRestorerDelegate <NSObject>
// Methods return YES if cancel is requested.
- (BOOL)glacierRestorerMessageDidChange:(NSString *)message;
- (BOOL)glacierRestorerBytesRequestedDidChange:(NSNumber *)theRequested;
- (BOOL)glacierRestorerTotalBytesToRequestDidChange:(NSNumber *)theMaxRequested;
- (BOOL)glacierRestorerDidFinishRequesting;
- (BOOL)glacierRestorerBytesTransferredDidChange:(NSNumber *)theTransferred;
- (BOOL)glacierRestorerTotalBytesToTransferDidChange:(NSNumber *)theTotal;
- (BOOL)glacierRestorerErrorMessage:(NSString *)theErrorMessage didOccurForPath:(NSString *)thePath;
- (BOOL)glacierRestorerDidSucceed;
- (BOOL)glacierRestorerDidFail:(NSError *)error;
@end

58
glacierrestore/GlacierRestorerParamSet.h Normal file
View file

@ -0,0 +1,58 @@
//
// GlacierRestorerParamSet.h
// Arq
//
// Created by Stefan Reitshamer on 5/29/13.
// Copyright (c) 2013 Stefan Reitshamer. All rights reserved.
//
@class AWSRegion;
@class BlobKey;
@class Bucket;
@interface GlacierRestorerParamSet : NSObject {
Bucket *bucket;
NSString *encryptionPassword;
double downloadBytesPerSecond;
BlobKey *commitBlobKey;
NSString *rootItemName;
int treeVersion;
BOOL treeIsCompressed;
BlobKey *treeBlobKey;
NSString *nodeName;
uid_t targetUID;
gid_t targetGID;
BOOL useTargetUIDAndGID;
NSString *destinationPath;
int logLevel;
}
- (id)initWithBucket:(Bucket *)theBucket
encryptionPassword:(NSString *)theEncryptionPassword
downloadBytesPerSecond:(double)theDownloadBytesPerSecond
commitBlobKey:(BlobKey *)theCommitBlobKey
rootItemName:(NSString *)theRootItemName
treeVersion:(int32_t)theTreeVersion
treeIsCompressed:(BOOL)theTreeIsCompressed
treeBlobKey:(BlobKey *)theTreeBlobKey
nodeName:(NSString *)theNodeName
targetUID:(uid_t)theTargetUID
targetGID:(gid_t)theTargetGID
useTargetUIDAndGID:(BOOL)theUseTargetUIDAndGID
destinationPath:(NSString *)theDestination
logLevel:(int)theLogLevel;
@property (readonly, retain) Bucket *bucket;
@property (readonly, retain) NSString *encryptionPassword;
@property (readonly) double downloadBytesPerSecond;
@property (readonly, retain) BlobKey *commitBlobKey;
@property (readonly, retain) NSString *rootItemName;
@property (readonly, retain) BlobKey *treeBlobKey;
@property (readonly, retain) NSString *nodeName;
@property (readonly) uid_t targetUID;
@property (readonly) gid_t targetGID;
@property (readonly) BOOL useTargetUIDAndGID;
@property (readonly, retain) NSString *destinationPath;
@property (readonly) int logLevel;
@end

79
glacierrestore/GlacierRestorerParamSet.m Normal file
View file

@ -0,0 +1,79 @@
//
// GlacierRestorerParamSet.m
// Arq
//
// Created by Stefan Reitshamer on 5/29/13.
// Copyright (c) 2013 Stefan Reitshamer. All rights reserved.
//
#import "GlacierRestorerParamSet.h"
#import "AWSRegion.h"
#import "BlobKey.h"
#import "StringIO.h"
#import "IntegerIO.h"
#import "BlobKeyIO.h"
#import "BooleanIO.h"
#import "DataIO.h"
#import "Tree.h"
#import "DoubleIO.h"
#import "Bucket.h"
@implementation GlacierRestorerParamSet
@synthesize bucket;
@synthesize encryptionPassword;
@synthesize downloadBytesPerSecond;
@synthesize commitBlobKey;
@synthesize rootItemName;
@synthesize treeBlobKey;
@synthesize nodeName;
@synthesize targetUID;
@synthesize targetGID;
@synthesize useTargetUIDAndGID;
@synthesize destinationPath;
@synthesize logLevel;
- (id)initWithBucket:(Bucket *)theBucket
encryptionPassword:(NSString *)theEncryptionPassword
downloadBytesPerSecond:(double)theDownloadBytesPerSecond
commitBlobKey:(BlobKey *)theCommitBlobKey
rootItemName:(NSString *)theRootItemName
treeVersion:(int32_t)theTreeVersion
treeIsCompressed:(BOOL)theTreeIsCompressed
treeBlobKey:(BlobKey *)theTreeBlobKey
nodeName:(NSString *)theNodeName
targetUID:(uid_t)theTargetUID
targetGID:(gid_t)theTargetGID
useTargetUIDAndGID:(BOOL)theUseTargetUIDAndGID
destinationPath:(NSString *)theDestination
logLevel:(int)theLogLevel {
if (self = [super init]) {
bucket = [theBucket retain];
encryptionPassword = [theEncryptionPassword retain];
downloadBytesPerSecond = theDownloadBytesPerSecond;
commitBlobKey = [theCommitBlobKey retain];
rootItemName = [theRootItemName retain];
treeVersion = theTreeVersion;
treeIsCompressed = theTreeIsCompressed;
treeBlobKey = [theTreeBlobKey retain];
nodeName = [theNodeName retain];
targetUID = theTargetUID;
targetGID = theTargetGID;
useTargetUIDAndGID = theUseTargetUIDAndGID;
destinationPath = [theDestination retain];
logLevel = theLogLevel;
}
return self;
}
- (void)dealloc {
[bucket release];
[encryptionPassword release];
[commitBlobKey release];
[rootItemName release];
[treeBlobKey release];
[nodeName release];
[destinationPath release];
[super dealloc];
}
@end

14
repo/BinarySHA1.h Normal file
View file

@ -0,0 +1,14 @@
//
// BinarySHA1.h
//
// Created by Stefan Reitshamer on 12/30/09.
// Copyright 2009 Haystack Software. All rights reserved.
//
@interface BinarySHA1 : NSObject {
}
+ (NSComparisonResult)compare:(const void *)a to:(const void *)b;
@end

25
repo/BinarySHA1.m Normal file
View file

@ -0,0 +1,25 @@
//
// BinarySHA1.m
//
// Created by Stefan Reitshamer on 12/30/09.
// Copyright 2009 Haystack Software. All rights reserved.
//
#import "BinarySHA1.h"
@implementation BinarySHA1
+ (NSComparisonResult)compare:(const void *)a to:(const void *)b {
unsigned char *left = (unsigned char *)a;
unsigned char *right = (unsigned char *)b;
for (int i = 0; i < 20; i++) {
if (left[i] < right[i]) {
return NSOrderedAscending;
}
if (left[i] > right[i]) {
return NSOrderedDescending;
}
}
return NSOrderedSame;
}
@end

50
s3glacierrestore/S3GlacierRestorer.h Normal file
View file

@ -0,0 +1,50 @@
//
// S3GlacierRestorer.h
// Arq
//
// Created by Stefan Reitshamer on 1/9/14.
// Copyright (c) 2014 Stefan Reitshamer. All rights reserved.
//
#import "TargetConnection.h"
#import "Restorer.h"
@class S3GlacierRestorerParamSet;
@protocol S3GlacierRestorerDelegate;
@class Repo;
@class Commit;
@class Tree;
@interface S3GlacierRestorer : NSObject <Restorer, TargetConnectionDelegate> {
S3GlacierRestorerParamSet *paramSet;
id <S3GlacierRestorerDelegate> delegate;
Repo *repo;
Commit *commit;
Tree *rootTree;
NSMutableArray *calculateItems;
NSMutableArray *glacierRequestItems;
NSMutableArray *restoreItems;
NSString *skipFilesRoot;
unsigned long long bytesActuallyRequestedThisRound;
unsigned long long bytesRequested;
unsigned long long totalBytesToRequest;
unsigned long long bytesTransferred;
unsigned long long totalBytesToTransfer;
unsigned long long bytesToRequestPerRound;
NSDate *dateToResumeRequesting;
NSUInteger roundsCompleted;
NSMutableDictionary *hardlinks;
NSUInteger sleepCycles;
}
- (id)initWithS3GlacierRestorerParamSet:(S3GlacierRestorerParamSet *)theParamSet delegate:(id <S3GlacierRestorerDelegate>)theDelegate;
- (void)run;
@end

445
s3glacierrestore/S3GlacierRestorer.m Normal file
View file

@ -0,0 +1,445 @@
//
// S3GlacierRestorer.m
// Arq
//
// Created by Stefan Reitshamer on 1/9/14.
// Copyright (c) 2014 Stefan Reitshamer. All rights reserved.
//
#import "S3GlacierRestorer.h"
#import "S3GlacierRestorerParamSet.h"
#import "S3GlacierRestorerDelegate.h"
#import "commit.h"
#import "Tree.h"
#import "Node.h"
#import "Repo.h"
#import "CalculateItem.h"
#import "GlacierRequestItem.h"
#import "RestoreItem.h"
#import "NSFileManager_extra.h"
#import "UserLibrary_Arq.h"
#import "BlobKey.h"
#import "Bucket.h"
#import "Target.h"
#import "S3Service.h"
#define RESTORE_DAYS (10)
#define SLEEP_CYCLES_START (1)
#define SLEEP_CYCLES_MAX (30)
#define SLEEP_CYCLE_DURATION (2.0)
@implementation S3GlacierRestorer
- (id)initWithS3GlacierRestorerParamSet:(S3GlacierRestorerParamSet *)theParamSet delegate:(id <S3GlacierRestorerDelegate>)theDelegate {
if (self = [super init]) {
paramSet = [theParamSet retain];
delegate = theDelegate;
calculateItems = [[NSMutableArray alloc] init];
glacierRequestItems = [[NSMutableArray alloc] init];
restoreItems = [[NSMutableArray alloc] init];
skipFilesRoot = [[[UserLibrary arqUserLibraryPath] stringByAppendingFormat:@"/RestoreJobSkipFiles/%f", [NSDate timeIntervalSinceReferenceDate]] retain];
bytesToRequestPerRound = paramSet.downloadBytesPerSecond * 60 * 60 * 4; // 4 hours at preferred download rate
dateToResumeRequesting = [[NSDate date] retain];
hardlinks = [[NSMutableDictionary alloc] init];
sleepCycles = SLEEP_CYCLES_START;
}
return self;
}
- (void)dealloc {
[paramSet release];
[repo release];
[commit release];
[rootTree release];
[calculateItems release];
[glacierRequestItems release];
[restoreItems release];
[skipFilesRoot release];
[dateToResumeRequesting release];
[hardlinks release];
[super dealloc];
}
- (NSString *)errorDomain {
return @"S3GlacierRestorerErrorDomain";
}
- (void)run {
NSError *myError = nil;
if (![self run:&myError]) {
[delegate s3GlacierRestorerDidFail:myError];
} else {
[delegate s3GlacierRestorerDidSucceed];
}
NSError *removeError = nil;
if ([[NSFileManager defaultManager] fileExistsAtPath:skipFilesRoot] && ![[NSFileManager defaultManager] removeItemAtPath:skipFilesRoot error:&removeError]) {
HSLogError(@"failed to remove %@: %@", skipFilesRoot, removeError);
}
HSLogDebug(@"S3GlacierRestorer finished");
}
#pragma mark Restorer
- (BOOL)requestBlobKey:(BlobKey *)theBlobKey error:(NSError **)error {
if (theBlobKey == nil) {
return YES;
}
unsigned long long dataSize = 0;
NSNumber *contains = [repo containsBlobForBlobKey:theBlobKey dataSize:&dataSize error:error];
if (contains == nil) {
return NO;
}
if (![contains boolValue]) {
// We'll report this to the user as an error during the download phase.
HSLogError(@"repo does not contain %@!", theBlobKey);
} else {
BOOL alreadyRestoredOrRestoring = NO;
if (![repo restoreObjectForBlobKey:theBlobKey forDays:RESTORE_DAYS alreadyRestoredOrRestoring:&alreadyRestoredOrRestoring error:error]) {
return NO;
}
unsigned long long actualBytesRequested = alreadyRestoredOrRestoring ? 0 : dataSize;
if (![self addToBytesRequested:dataSize actualBytesRequested:actualBytesRequested error:error]) {
return NO;
}
}
return YES;
}
- (NSNumber *)isObjectAvailableForBlobKey:(BlobKey *)theBlobKey error:(NSError **)error {
return [repo isObjectDownloadableForBlobKey:theBlobKey error:error];
}
- (NSNumber *)sizeOfBlob:(BlobKey *)theBlobKey error:(NSError **)error {
unsigned long long size = 0;
NSNumber *contains = [repo containsBlobForBlobKey:theBlobKey dataSize:&size error:error];
if (contains == nil) {
return nil;
}
if (![contains boolValue]) {
SETNSERROR([self errorDomain], ERROR_NOT_FOUND, @"size of blob %@ not found because blob not found", theBlobKey);
return NO;
}
return [NSNumber numberWithUnsignedLongLong:size];
}
- (NSData *)dataForBlobKey:(BlobKey *)theBlobKey error:(NSError **)error {
NSData *data = [repo dataForBlobKey:theBlobKey error:error];
if (data == nil) {
return nil;
}
if (![self addToBytesTransferred:(unsigned long long)[data length] error:error]) {
return nil;
}
return data;
}
- (BOOL)shouldSkipFile:(NSString *)thePath {
NSString *skipFilePath = [skipFilesRoot stringByAppendingString:thePath];
return [[NSFileManager defaultManager] fileExistsAtPath:skipFilePath];
}
- (BOOL)useTargetUIDAndGID {
return paramSet.useTargetUIDAndGID;
}
- (uid_t)targetUID {
return paramSet.targetUID;
}
- (gid_t)targetGID {
return paramSet.targetGID;
}
#pragma mark internal
- (BOOL)run:(NSError **)error {
repo = [[Repo alloc] initWithBucket:[paramSet bucket]
encryptionPassword:[paramSet encryptionPassword]
targetUID:[paramSet targetUID]
targetGID:[paramSet targetGID]
loadExistingMutablePackFiles:NO
targetConnectionDelegate:self
repoDelegate:nil
error:error];
if (repo == nil) {
return NO;
}
commit = [[repo commitForBlobKey:[paramSet commitBlobKey] dataSize:NULL error:error] retain];
if (commit == nil) {
return NO;
}
rootTree = [[repo treeForBlobKey:[paramSet treeBlobKey] dataSize:NULL error:error] retain];
if (rootTree == nil) {
return NO;
}
if (paramSet.nodeName != nil) {
Node *node = [rootTree childNodeWithName:paramSet.nodeName];
if ([[rootTree childNodeNames] isEqualToArray:[NSArray arrayWithObject:@"."]]) {
// The single-file case.
node = [rootTree childNodeWithName:@"."];
}
NSAssert(node != nil, @"node can't be nil");
[calculateItems addObject:[[[CalculateItem alloc] initWithPath:paramSet.destinationPath node:node] autorelease]];
[glacierRequestItems addObject:[[[GlacierRequestItem alloc] initWithPath:paramSet.destinationPath node:node] autorelease]];
[restoreItems addObject:[[[RestoreItem alloc] initWithPath:paramSet.destinationPath tree:rootTree node:node] autorelease]];
} else {
[calculateItems addObject:[[[CalculateItem alloc] initWithPath:paramSet.destinationPath tree:rootTree] autorelease]];
[glacierRequestItems addObject:[[[GlacierRequestItem alloc] initWithPath:paramSet.destinationPath tree:rootTree] autorelease]];
[restoreItems addObject:[[[RestoreItem alloc] initWithPath:paramSet.destinationPath tree:rootTree] autorelease]];
}
NSString *calculatingMessage = @"Calculating sizes";
if ([[NSFileManager defaultManager] fileExistsAtPath:paramSet.destinationPath]) {
calculatingMessage = @"Comparing existing files to backup data";
}
if ([delegate s3GlacierRestorerMessageDidChange:calculatingMessage]) {
SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested");
return NO;
}
if (![self calculateSizes:error]) {
return NO;
}
NSDateFormatter *dateFormatter = [[[NSDateFormatter alloc] init] autorelease];
[dateFormatter setDateStyle:NSDateFormatterMediumStyle];
[dateFormatter setTimeStyle:NSDateFormatterShortStyle];
NSString *commitDescription = [dateFormatter stringFromDate:[commit creationDate]];
if ([delegate s3GlacierRestorerMessageDidChange:[NSString stringWithFormat:@"Restoring %@ from %@ to %@", paramSet.rootItemName, commitDescription, paramSet.destinationPath]]) {
SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested");
return NO;
}
BOOL restoredAnItem = NO;
BOOL ret = YES;
NSAutoreleasePool *pool = nil;
while ([restoreItems count] > 0) {
[pool drain];
pool = [[NSAutoreleasePool alloc] init];
if ([glacierRequestItems count] > 0) {
// Reset counters if necessary.
if (bytesActuallyRequestedThisRound >= bytesToRequestPerRound) {
roundsCompleted++;
bytesActuallyRequestedThisRound = 0;
NSDate *nextResumeDate = [[dateToResumeRequesting dateByAddingTimeInterval:(60 * 60 * 4)] retain];
[dateToResumeRequesting release];
dateToResumeRequesting = nextResumeDate;
HSLogDebug(@"reset next request resume date to %@", nextResumeDate);
}
// Make sure we've transferred all the bytes from all but the most recent round of requests.
double theMinimum = (roundsCompleted = 0) ? 0 : ((double)bytesToRequestPerRound * (double)(roundsCompleted - 1)) * .9;
unsigned long long minimumBytesToHaveTransferred = (unsigned long long)theMinimum;
if ((bytesActuallyRequestedThisRound < bytesToRequestPerRound)
&& (bytesTransferred >= minimumBytesToHaveTransferred)
&& ([[NSDate date] earlierDate:dateToResumeRequesting] == dateToResumeRequesting)) {
// Request more Glacier items.
if (![self requestMoreGlacierItems:error]) {
ret = NO;
break;
}
}
if ([glacierRequestItems count] == 0) {
HSLogDebug(@"finished requesting");
if ([delegate s3GlacierRestorerDidFinishRequesting]) {
SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested");
ret = NO;
break;
}
}
}
// Restore an item if possible.
NSError *restoreError = nil;
RestoreItem *restoreItem = [restoreItems objectAtIndex:0];
restoredAnItem = YES;
HSLogDebug(@"attempting to restore %@", restoreItem);
if (![restoreItem restoreWithHardlinks:hardlinks restorer:self error:&restoreError]) {
if ([restoreError isErrorWithDomain:[restoreItem errorDomain] code:ERROR_GLACIER_OBJECT_NOT_AVAILABLE]) {
HSLogDebug(@"glacier object not available yet");
restoredAnItem = NO;
} else if ([restoreError isErrorWithDomain:[self errorDomain] code:ERROR_ABORT_REQUESTED]) {
if (error != NULL) {
*error = restoreError;
}
ret = NO;
break;
} else {
[delegate s3GlacierRestorerErrorMessage:[restoreError localizedDescription] didOccurForPath:[restoreItem path]];
}
}
if (restoredAnItem) {
NSArray *nextItems = [restoreItem nextItemsWithRepo:repo error:error];
if (nextItems == nil) {
ret = NO;
break;
}
[restoreItems removeObjectAtIndex:0];
if ([nextItems count] > 0) {
[restoreItems insertObjects:nextItems atIndexes:[NSIndexSet indexSetWithIndexesInRange:NSMakeRange(0, [nextItems count])]];
}
}
if (!restoredAnItem) {
HSLogDebug(@"sleeping");
for (NSUInteger i = 0; i < sleepCycles; i++) {
if (![self addToBytesTransferred:0 error:error]) {
ret = NO;
break;
}
[NSThread sleepForTimeInterval:SLEEP_CYCLE_DURATION];
}
sleepCycles *= 2;
if (sleepCycles > SLEEP_CYCLES_MAX) {
sleepCycles = SLEEP_CYCLES_MAX;
}
}
if (!ret) {
break;
}
}
if (!ret && error != NULL) {
[*error retain];
}
[pool drain];
if (!ret && error != NULL) {
[*error autorelease];
}
return ret;
}
- (BOOL)calculateSizes:(NSError **)error {
BOOL ret = YES;
NSAutoreleasePool *pool = nil;
while ([calculateItems count] > 0) {
[pool drain];
pool = [[NSAutoreleasePool alloc] init];
CalculateItem *item = [calculateItems objectAtIndex:0];
if (![item calculateWithRepo:repo restorer:self error:error]) {
ret = NO;
break;
}
for (NSString *path in [item filesToSkip]) {
[self skipFile:path];
}
unsigned long long bytesToTransfer = [item bytesToTransfer];
if (![self addToTotalBytesToRequest:bytesToTransfer error:error]) {
ret = NO;
break;
}
if (![self addToTotalBytesToTransfer:bytesToTransfer error:error]) {
ret = NO;
break;
}
[calculateItems removeObjectAtIndex:0];
NSArray *nextItems = [item nextItems];
if ([nextItems count] > 0) {
[calculateItems insertObjects:nextItems atIndexes:[NSIndexSet indexSetWithIndexesInRange:NSMakeRange(0, [nextItems count])]];
}
}
if (!ret && error != NULL) {
[*error retain];
}
[pool drain];
if (!ret && error != NULL) {
[*error autorelease];
}
return ret;
}
- (void)skipFile:(NSString *)thePath {
NSString *skipFilePath = [skipFilesRoot stringByAppendingString:thePath];
NSError *myError = nil;
if (![[NSFileManager defaultManager] ensureParentPathExistsForPath:skipFilePath targetUID:paramSet.targetUID targetGID:paramSet.targetGID error:&myError]) {
HSLogError(@"error creating parent dir for %@: %@", skipFilePath, myError);
return;
}
if (![[NSFileManager defaultManager] touchFileAtPath:skipFilePath targetUID:paramSet.targetUID targetGID:paramSet.targetGID error:&myError]) {
HSLogError(@"error touching %@: %@", skipFilePath, myError);
}
HSLogDebug(@"skip file %@", thePath);
}
- (BOOL)requestMoreGlacierItems:(NSError **)error {
BOOL ret = YES;
NSAutoreleasePool *pool = nil;
while (bytesActuallyRequestedThisRound < bytesToRequestPerRound && [glacierRequestItems count] > 0) {
[pool drain];
pool = [[NSAutoreleasePool alloc] init];
GlacierRequestItem *item = [glacierRequestItems objectAtIndex:0];
NSArray *nextItems = [item requestWithRestorer:self repo:repo error:error];
if (nextItems == nil) {
ret = NO;
break;
}
[glacierRequestItems removeObjectAtIndex:0];
if ([nextItems count] > 0) {
[glacierRequestItems insertObjects:nextItems atIndexes:[NSIndexSet indexSetWithIndexesInRange:NSMakeRange(0, [nextItems count])]];
}
}
if (!ret && error != NULL) {
[*error retain];
}
[pool drain];
if (!ret && error != NULL) {
[*error autorelease];
}
return ret;
}
- (BOOL)addToBytesRequested:(unsigned long long)length actualBytesRequested:(unsigned long long)actualBytesRequested error:(NSError **)error {
bytesRequested += length;
bytesActuallyRequestedThisRound += actualBytesRequested;
if ([delegate s3GlacierRestorerBytesRequestedDidChange:[NSNumber numberWithUnsignedLongLong:bytesRequested]]) {
SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested");
return NO;
}
return YES;
}
- (BOOL)addToTotalBytesToRequest:(unsigned long long)length error:(NSError **)error {
totalBytesToRequest += length;
if ([delegate s3GlacierRestorerTotalBytesToRequestDidChange:[NSNumber numberWithUnsignedLongLong:totalBytesToRequest]]) {
SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested");
return NO;
}
return YES;
}
- (BOOL)addToBytesTransferred:(unsigned long long)length error:(NSError **)error {
bytesTransferred += length;
if ([delegate s3GlacierRestorerBytesTransferredDidChange:[NSNumber numberWithUnsignedLongLong:bytesTransferred]]) {
SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested");
return NO;
}
return YES;
}
- (BOOL)addToTotalBytesToTransfer:(unsigned long long)length error:(NSError **)error {
totalBytesToTransfer += length;
if ([delegate s3GlacierRestorerTotalBytesToTransferDidChange:[NSNumber numberWithUnsignedLongLong:totalBytesToTransfer]]) {
SETNSERROR([self errorDomain], ERROR_ABORT_REQUESTED, @"cancel requested");
return NO;
}
return YES;
}
#pragma mark TargetConnectionDelegate
- (BOOL)targetConnectionShouldRetryOnTransientError:(NSError **)error {
return YES;
}
@end

28
s3glacierrestore/S3GlacierRestorerDelegate.h Normal file
View file

@ -0,0 +1,28 @@
//
// S3GlacierRestorerDelegate.h
// Arq
//
// Created by Stefan Reitshamer on 1/9/14.
// Copyright (c) 2014 Stefan Reitshamer. All rights reserved.
//
@protocol S3GlacierRestorerDelegate <NSObject>
// Methods return YES if cancel is requested.
- (BOOL)s3GlacierRestorerMessageDidChange:(NSString *)message;
- (BOOL)s3GlacierRestorerBytesRequestedDidChange:(NSNumber *)theRequested;
- (BOOL)s3GlacierRestorerTotalBytesToRequestDidChange:(NSNumber *)theMaxRequested;
- (BOOL)s3GlacierRestorerDidFinishRequesting;
- (BOOL)s3GlacierRestorerBytesTransferredDidChange:(NSNumber *)theTransferred;
- (BOOL)s3GlacierRestorerTotalBytesToTransferDidChange:(NSNumber *)theTotal;
- (BOOL)s3GlacierRestorerErrorMessage:(NSString *)theErrorMessage didOccurForPath:(NSString *)thePath;
- (void)s3GlacierRestorerDidSucceed;
- (void)s3GlacierRestorerDidFail:(NSError *)error;
@end

61
s3glacierrestore/S3GlacierRestorerParamSet.h Normal file
View file

@ -0,0 +1,61 @@
//
// S3GlacierRestorerParamSet.h
// Arq
//
// Created by Stefan Reitshamer on 1/9/14.
// Copyright (c) 2014 Stefan Reitshamer. All rights reserved.
//
@class BufferedInputStream;
@class BufferedOutputStream;
@class AWSRegion;
@class BlobKey;
@class Bucket;
@interface S3GlacierRestorerParamSet : NSObject {
Bucket *bucket;
NSString *encryptionPassword;
double downloadBytesPerSecond;
BlobKey *commitBlobKey;
NSString *rootItemName;
int treeVersion;
BOOL treeIsCompressed;
BlobKey *treeBlobKey;
NSString *nodeName;
uid_t targetUID;
gid_t targetGID;
BOOL useTargetUIDAndGID;
NSString *destinationPath;
int logLevel;
}
- (id)initWithBucket:(Bucket *)theBucket
encryptionPassword:(NSString *)theEncryptionPassword
downloadBytesPerSecond:(double)theDownloadBytesPerSecond
commitBlobKey:(BlobKey *)theCommitBlobKey
rootItemName:(NSString *)theRootItemName
treeVersion:(int32_t)theTreeVersion
treeIsCompressed:(BOOL)theTreeIsCompressed
treeBlobKey:(BlobKey *)theTreeBlobKey
nodeName:(NSString *)theNodeName
targetUID:(uid_t)theTargetUID
targetGID:(gid_t)theTargetGID
useTargetUIDAndGID:(BOOL)theUseTargetUIDAndGID
destinationPath:(NSString *)theDestination
logLevel:(int)theLogLevel;
@property (readonly, retain) Bucket *bucket;
@property (readonly, retain) NSString *encryptionPassword;
@property (readonly) double downloadBytesPerSecond;
@property (readonly, retain) BlobKey *commitBlobKey;
@property (readonly, retain) NSString *rootItemName;
@property (readonly, retain) BlobKey *treeBlobKey;
@property (readonly, retain) NSString *nodeName;
@property (readonly) uid_t targetUID;
@property (readonly) gid_t targetGID;
@property (readonly) BOOL useTargetUIDAndGID;
@property (readonly, retain) NSString *destinationPath;
@property (readonly) int logLevel;
@end

79
s3glacierrestore/S3GlacierRestorerParamSet.m Normal file
View file

@ -0,0 +1,79 @@
//
// S3GlacierRestorerParamSet.m
// Arq
//
// Created by Stefan Reitshamer on 1/9/14.
// Copyright (c) 2014 Stefan Reitshamer. All rights reserved.
//
#import "S3GlacierRestorerParamSet.h"
#import "AWSRegion.h"
#import "BlobKey.h"
#import "StringIO.h"
#import "IntegerIO.h"
#import "BlobKeyIO.h"
#import "BooleanIO.h"
#import "DataIO.h"
#import "Tree.h"
#import "DoubleIO.h"
#import "Bucket.h"
@implementation S3GlacierRestorerParamSet
@synthesize bucket;
@synthesize encryptionPassword;
@synthesize downloadBytesPerSecond;
@synthesize commitBlobKey;
@synthesize rootItemName;
@synthesize treeBlobKey;
@synthesize nodeName;
@synthesize targetUID;
@synthesize targetGID;
@synthesize useTargetUIDAndGID;
@synthesize destinationPath;
@synthesize logLevel;
- (id)initWithBucket:(Bucket *)theBucket
encryptionPassword:(NSString *)theEncryptionPassword
downloadBytesPerSecond:(double)theDownloadBytesPerSecond
commitBlobKey:(BlobKey *)theCommitBlobKey
rootItemName:(NSString *)theRootItemName
treeVersion:(int32_t)theTreeVersion
treeIsCompressed:(BOOL)theTreeIsCompressed
treeBlobKey:(BlobKey *)theTreeBlobKey
nodeName:(NSString *)theNodeName
targetUID:(uid_t)theTargetUID
targetGID:(gid_t)theTargetGID
useTargetUIDAndGID:(BOOL)theUseTargetUIDAndGID
destinationPath:(NSString *)theDestination
logLevel:(int)theLogLevel {
if (self = [super init]) {
bucket = [theBucket retain];
encryptionPassword = [theEncryptionPassword retain];
downloadBytesPerSecond = theDownloadBytesPerSecond;
commitBlobKey = [theCommitBlobKey retain];
rootItemName = [theRootItemName retain];
treeVersion = theTreeVersion;
treeIsCompressed = theTreeIsCompressed;
treeBlobKey = [theTreeBlobKey retain];
nodeName = [theNodeName retain];
targetUID = theTargetUID;
targetGID = theTargetGID;
useTargetUIDAndGID = theUseTargetUIDAndGID;
destinationPath = [theDestination retain];
logLevel = theLogLevel;
}
return self;
}
- (void)dealloc {
[bucket release];
[encryptionPassword release];
[commitBlobKey release];
[rootItemName release];
[treeBlobKey release];
[nodeName release];
[destinationPath release];
[super dealloc];
}
@end