Merge branch '1SecondEveryday:main' into main

Package.swift

@@ -23,6 +23,10 @@ let package = Package(
             dependencies: []),
         .testTarget(
             name: "OsirisTests",
-            dependencies: ["Osiris"]),
+            dependencies: ["Osiris"],
+            resources: [
+                .copy("Resources/lorem.txt"),
+                .copy("Resources/notbad.jpg"),
+            ]),
     ]
 )

Readme.md

@@ -14,20 +14,20 @@ Create an encoder and then add parts to it as needed:
 ```Swift
 let avatarData = UIImage(from: somewhere).jpegData(compressionQuality: 1)
 let encoder = MultipartFormEncoder()
-let body = encoder.encode(parts: [
+let body = try encoder.encodeData(parts: [
     .text(name: "email", text: "somebody@example.com"),
     .text(name: "password", text: "secret"),
     .binary(name: "avatar", type: "image/jpeg", data: avatarData, filename: "avatar.jpg"),
 ])
 ```
 
-The entire form is encoded as `Data` in memory so you may not want to use this for more than a few megabytes at a time:
+The form can be encoded as `Data` in memory, or to a file. There's a hard limit of 50 MB on encoding to memory but in practice you probably never want to go that high purely in memory. If you're adding any kind of image or video file then it's probably better to stream to a file.
 
 ```Swift
-let body = encoder.encode(parts: [/* ... */])
+let body = try encoder.encodeFile(parts: [/* ... */])
 var request = URLRequest(url: URL(string: "https://example.com/accounts")!)
 request.httpMethod = "POST"
-request.httpBody = body.data
+request.httpBodyStream = InputStream(url: body.url)
 request.addValue(body.contentType, forHTTPHeaderField: "Content-Type")
 request.addValue("\(body.contentLength)", forHTTPHeaderField: "Content-Length")
 // ... whatever you normally do with requests

Sources/Osiris/MultipartFormEncoder.swift

@@ -24,7 +24,7 @@
 import Foundation
 
 extension MultipartFormEncoder {
-    struct Body {
+    struct BodyData {
         let contentType: String
         let data: Data
 
@@ -33,62 +33,194 @@ extension MultipartFormEncoder {
         }
     }
 
-    struct Part {
-        enum Content {
+    struct BodyFile {
+        let contentType: String
+        let url: URL
+        let contentLength: Int64
+    }
+
+    struct Part: Equatable {
+        enum Content: Equatable {
             case text(String)
-            case binary(Data, type: String, filename: String)
+            case binaryData(Data, type: String, filename: String)
+            case binaryFile(URL, size: Int64, type: String, filename: String)
         }
 
         let name: String
         let content: Content
 
-        static func text(name: String, value: String) -> Part {
+        static func text(_ value: String, name: String) -> Part {
             Part(name: name, content: .text(value))
         }
 
-        static func binary(name: String, data: Data, type: String, filename: String) -> Part {
-            Part(name: name, content: .binary(data, type: type, filename: filename))
+        static func data(_ data: Data, name: String, type: String, filename: String) -> Part {
+            Part(name: name, content: .binaryData(data, type: type, filename: filename))
+        }
+
+        static func file(_ url: URL, name: String, type: String, filename: String? = nil) throws -> Part {
+            let attributes = try FileManager.default.attributesOfItem(atPath: url.path)
+            guard let size = attributes[.size] as? Int64 else {
+                throw Error.invalidFile(url)
+            }
+            return Part(name: name, content: .binaryFile(url, size: size, type: type, filename: filename ?? url.lastPathComponent))
         }
     }
 }
 
 final class MultipartFormEncoder {
+    enum Error: Swift.Error {
+        case invalidFile(URL)
+        case invalidOutputFile(URL)
+        case streamError
+        case tooMuchDataForMemory
+    }
+
     let boundary: String
 
     init(boundary: String? = nil) {
-        self.boundary = boundary ?? "LifeIsMadeOfSeconds-\(UUID().uuidString)"
+        self.boundary = boundary ?? "Osiris-\(UUID().uuidString)"
     }
 
-    func encode(parts: [Part]) -> Body {
-        var bodyData = Data()
-        for part in parts {
-            // Header
-            bodyData.append(Data("--\(boundary)\r\n".utf8))
-            switch part.content {
-            case .text:
-                bodyData.append(Data("Content-Disposition: form-data; name=\"\(part.name)\"\r\n".utf8))
-
-            case let .binary(data, type, filename):
-                bodyData.append(Data("Content-Disposition: form-data; name=\"\(part.name)\"; filename=\"\(filename)\"\r\n".utf8))
-                bodyData.append(Data("Content-Type: \(type)\r\n".utf8))
-                bodyData.append(Data("Content-Length: \(data.count)\r\n".utf8))
-            }
-            bodyData.append(Data("\r\n".utf8))
-
-            // Body
+    func encodeData(parts: [Part]) throws -> BodyData {
+        let totalSize: Int64 = parts.reduce(0, { size, part in
             switch part.content {
             case let .text(string):
-                bodyData.append(Data(string.utf8))
+                return size + Int64(string.lengthOfBytes(using: .utf8))
 
-            case let .binary(data, _, _):
-                bodyData.append(data)
+            case let .binaryData(data, _, _):
+                return size + Int64(data.count)
+
+            case let .binaryFile(_, fileSize, _, _):
+                return size + fileSize
             }
-            bodyData.append(Data("\r\n".utf8))
+        })
+        guard totalSize < 50_000_000 else {
+            throw Error.tooMuchDataForMemory
+        }
+
+        let stream = OutputStream(toMemory: ())
+        stream.open()
+
+        for part in parts {
+            try encodePart(part, to: stream)
         }
 
         // Footer
-        bodyData.append(Data("--\(boundary)--".utf8))
+        try encode(string: "--\(boundary)--", to: stream)
 
-        return Body(contentType: "multipart/form-data; boundary=\"\(boundary)\"", data: bodyData)
+        stream.close()
+
+        guard let bodyData = stream.property(forKey: .dataWrittenToMemoryStreamKey) as? Data else {
+            throw Error.streamError
+        }
+        return BodyData(contentType: "multipart/form-data; boundary=\"\(boundary)\"", data: bodyData)
+    }
+
+    func encodeFile(parts: [Part]) throws -> BodyFile {
+        let fm = FileManager.default
+        let outputURL = tempFileURL()
+        guard let stream = OutputStream(url: outputURL, append: false) else {
+            _ = try? fm.removeItem(at: outputURL)
+            throw Error.invalidFile(outputURL)
+        }
+        stream.open()
+
+        for part in parts {
+            try encodePart(part, to: stream)
+        }
+
+        // Footer
+        try encode(string: "--\(boundary)--", to: stream)
+
+        stream.close()
+
+        let attributes = try FileManager.default.attributesOfItem(atPath: outputURL.path)
+        guard let size = attributes[.size] as? Int64 else {
+            throw Error.invalidOutputFile(outputURL)
+        }
+        let contentType = "multipart/form-data; boundary=\(boundary)"
+        return BodyFile(contentType: contentType, url: outputURL, contentLength: size)
+    }
+
+    private func tempFileURL() -> URL {
+        let timestamp = Int(Date().timeIntervalSince1970)
+        let filename = "multipart-\(timestamp)-\(Int.random(in: 0 ... .max))"
+        let url = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent(filename)
+        return url
+    }
+
+    private func encodePart(_ part: Part, to stream: OutputStream) throws {
+        // Header
+        try encode(string: "--\(boundary)\r\n", to: stream)
+        switch part.content {
+        case .text:
+            try encode(string: "Content-Disposition: form-data; name=\"\(part.name)\"\r\n", to: stream)
+
+        case let .binaryData(data, type, filename):
+            try encode(string: "Content-Disposition: form-data; name=\"\(part.name)\"; filename=\"\(filename)\"\r\n", to: stream)
+            try encode(string: "Content-Type: \(type)\r\n", to: stream)
+            try encode(string: "Content-Length: \(data.count)\r\n", to: stream)
+
+        case let .binaryFile(_, size, type, filename):
+            try encode(string: "Content-Disposition: form-data; name=\"\(part.name)\"; filename=\"\(filename)\"\r\n", to: stream)
+            try encode(string: "Content-Type: \(type)\r\n", to: stream)
+            try encode(string: "Content-Length: \(size)\r\n", to: stream)
+        }
+        try encode(string: "\r\n", to: stream)
+
+        // Body
+        switch part.content {
+        case let .text(string):
+            try encode(string: string, to: stream)
+
+        case let .binaryData(data, _, _):
+            try encode(data: data, to: stream)
+
+        case let .binaryFile(url, _, _, _):
+            try encode(url: url, to: stream)
+        }
+        try encode(string: "\r\n", to: stream)
+    }
+
+    private func encode(data: Data, to stream: OutputStream) throws {
+        guard !data.isEmpty else { return }
+
+        try data.withUnsafeBytes { (bytes: UnsafeRawBufferPointer) in
+            let uint8Bytes = bytes.baseAddress!.bindMemory(to: UInt8.self, capacity: bytes.count)
+            let written = stream.write(uint8Bytes, maxLength: bytes.count)
+            if written != bytes.count {
+                throw Error.streamError
+            }
+        }
+    }
+
+    private func encode(string: String, to stream: OutputStream) throws {
+        try encode(data: Data(string.utf8), to: stream)
+    }
+
+    private func encode(url: URL, to stream: OutputStream) throws {
+        guard let inStream = InputStream(url: url) else {
+            throw Error.streamError
+        }
+        let bufferSize = 128 * 1024
+        let buffer = UnsafeMutablePointer<UInt8>.allocate(capacity: bufferSize)
+        inStream.open()
+
+        defer {
+            buffer.deallocate()
+            inStream.close()
+        }
+
+        while inStream.hasBytesAvailable {
+            let bytesRead = inStream.read(buffer, maxLength: bufferSize)
+            guard bytesRead > 0 else {
+                break
+            }
+
+            let bytesWritten = stream.write(buffer, maxLength: bytesRead)
+            if bytesWritten != bytesRead {
+                throw Error.streamError
+            }
+        }
     }
 }

Sources/Osiris/RequestBuilder.swift

@@ -38,7 +38,7 @@ final class RequestBuilder {
 
             case .multipart:
                 let encoder = MultipartFormEncoder()
-                let body = encoder.encode(parts: request.parts)
+                let body = try encoder.encodeData(parts: request.parts)
                 result.addValue(body.contentType, forHTTPHeaderField: "Content-Type")
                 result.addValue("\(body.contentLength)", forHTTPHeaderField: "Content-Length")
                 result.httpBody = body.data

Tests/OsirisTests/MultipartFormEncoderTests.swift

@@ -1,6 +1,6 @@
 //
 //  MultipartFormEncoderTests.swift
-//  VidjoTests
+//  OsirisTests
 //
 //  Created by Sami Samhuri on 2020-10-20.
 //  Copyright © 2020 Guru Logic Inc. All rights reserved.
@@ -9,35 +9,60 @@
 @testable import Osiris
 import XCTest
 
-func AssertBodyEqual(_ expression1: @autoclosure () throws -> Data, _ expression2: @autoclosure () throws -> String, _ message: @autoclosure () -> String? = nil, file: StaticString = #filePath, line: UInt = #line) {
-    let data1 = try! expression1()
+func AssertStringDataEqual(_ expression1: @autoclosure () throws -> Data, _ expression2: @autoclosure () throws -> String, _ message: @autoclosure () -> String? = nil, file: StaticString = #filePath, line: UInt = #line) throws {
+    let data1 = try expression1()
     let string1 = String(bytes: data1, encoding: .utf8)!
-    let string2 = try! expression2()
+    let string2 = try expression2()
     let message = message() ?? "\"\(string1)\" is not equal to \"\(string2)\""
     XCTAssertEqual(data1, Data(string2.utf8), message, file: file, line: line)
 }
 
 class MultipartFormEncoderTests: XCTestCase {
-    var boundary = "SuperAwesomeBoundary"
     var subject: MultipartFormEncoder!
 
     override func setUpWithError() throws {
-        subject = MultipartFormEncoder(boundary: boundary)
+        subject = MultipartFormEncoder(boundary: "SuperAwesomeBoundary")
     }
 
     override func tearDownWithError() throws {
         subject = nil
     }
 
+    func testConstructTextPart() {
+        let part = MultipartFormEncoder.Part.text("value", name: "name")
+        XCTAssertEqual(part, MultipartFormEncoder.Part(name: "name", content: .text("value")))
+    }
+
+    func testConstructDataPart() {
+        let data = Data("value".utf8)
+        let part = MultipartFormEncoder.Part.data(data, name: "name", type: "text/plain", filename: "something.txt")
+        let expected = MultipartFormEncoder.Part(name: "name", content: .binaryData(data, type: "text/plain", filename: "something.txt"))
+        XCTAssertEqual(part, expected)
+    }
+
+    func testConstructBinaryFilePart() throws {
+        let url = Bundle.module.url(forResource: "notbad", withExtension: "jpg")!
+        let part = try MultipartFormEncoder.Part.file(url, name: "name", type: "image/jpeg")
+        let expected = MultipartFormEncoder.Part(name: "name", content: .binaryFile(url, size: 22_680, type: "image/jpeg", filename: "notbad.jpg"))
+        XCTAssertEqual(part, expected)
+    }
+
+    func testConstructInvalidFilePart() {
+        let url = Bundle.module.url(forResource: "notbad", withExtension: "jpg")!
+            .appendingPathComponent("busted")
+        XCTAssertThrowsError(try MultipartFormEncoder.Part.file(url, name: "name", type: "image/jpeg"))
+    }
+
     func testEncodeNothing() throws {
-        let body = subject.encode(parts: [])
+        let body = try subject.encodeData(parts: [])
         XCTAssertEqual(body.contentType, "multipart/form-data; boundary=\"SuperAwesomeBoundary\"")
-        AssertBodyEqual(body.data, "--SuperAwesomeBoundary--")
+        XCTAssertEqual(body.contentLength, 24)
+        try AssertStringDataEqual(body.data, "--SuperAwesomeBoundary--")
     }
 
     func testEncodeText() throws {
-        AssertBodyEqual(
-            subject.encode(parts: [.text(name: "name", value: "Tina")]).data,
+        try AssertStringDataEqual(
+            try subject.encodeData(parts: [.text("Tina", name: "name")]).data,
             [
                 "--SuperAwesomeBoundary",
                 "Content-Disposition: form-data; name=\"name\"",
@@ -48,11 +73,24 @@ class MultipartFormEncoderTests: XCTestCase {
         )
     }
 
+    func testEncodeEmptyText() throws {
+        try AssertStringDataEqual(
+            try subject.encodeData(parts: [.text("", name: "name")]).data,
+            [
+                "--SuperAwesomeBoundary",
+                "Content-Disposition: form-data; name=\"name\"",
+                "",
+                "",
+                "--SuperAwesomeBoundary--",
+            ].joined(separator: "\r\n")
+        )
+    }
+
     func testEncodeData() throws {
         let data = Data("phony video data".utf8)
-        AssertBodyEqual(
-            subject.encode(parts: [
-                .binary(name: "video", data: data, type: "video/mp4", filename: "LiesSex&VideoTape.mp4"),
+        try AssertStringDataEqual(
+            try subject.encodeData(parts: [
+                .data(data, name: "video", type: "video/mp4", filename: "LiesSex&VideoTape.mp4"),
             ]).data,
             [
                 "--SuperAwesomeBoundary",
@@ -61,20 +99,42 @@ class MultipartFormEncoderTests: XCTestCase {
                 "Content-Length: 16",
                 "",
                 "phony video data",
-                "--SuperAwesomeBoundary--"
+                "--SuperAwesomeBoundary--",
             ].joined(separator: "\r\n")
         )
     }
 
+    func testEncodeFile() throws {
+        let url = Bundle.module.url(forResource: "lorem", withExtension: "txt")!
+        let body = try subject.encodeFile(parts: [
+            .file(url, name: "lorem", type: "text/plain"),
+        ])
+        XCTAssertEqual(body.contentType, "multipart/form-data; boundary=SuperAwesomeBoundary")
+        XCTAssertEqual(body.contentLength, 3586)
+        XCTAssertEqual(try FileManager.default.attributesOfItem(atPath: body.url.path)[.size] as! UInt64, 3586)
+        XCTAssertEqual(try String(contentsOf: body.url), [
+            "--SuperAwesomeBoundary",
+            "Content-Disposition: form-data; name=\"lorem\"; filename=\"lorem.txt\"",
+            "Content-Type: text/plain",
+            "Content-Length: 3418",
+            "",
+            try! String(contentsOf: url),
+
+            "--SuperAwesomeBoundary--",
+        ].joined(separator: "\r\n"))
+    }
+
     func testEncodeEverything() throws {
         let imageData = Data("phony image data".utf8)
         let videoData = Data("phony video data".utf8)
-        AssertBodyEqual(
-            subject.encode(parts: [
-                .text(name: "name", value: "Queso"),
-                .binary(name: "image", data: imageData, type: "image/jpeg", filename: "feltcute.jpg"),
-                .text(name: "spot", value: "top of the bbq"),
-                .binary(name: "video", data: videoData, type: "video/mp4", filename: "LiesSex&VideoTape.mp4"),
+        let url = Bundle.module.url(forResource: "lorem", withExtension: "txt")!
+        try AssertStringDataEqual(
+            try subject.encodeData(parts: [
+                .text("Queso", name: "name"),
+                .data(imageData, name: "image", type: "image/jpeg", filename: "feltcute.jpg"),
+                .text("top of the bbq", name: "spot"),
+                .data(videoData, name: "video", type: "video/mp4", filename: "LiesSex&VideoTape.mp4"),
+                .file(url, name: "lorem", type: "text/plain"),
             ]).data,
             [
                 "--SuperAwesomeBoundary",
@@ -101,15 +161,15 @@ class MultipartFormEncoderTests: XCTestCase {
                 "",
                 "phony video data",
 
-                "--SuperAwesomeBoundary--"
+                "--SuperAwesomeBoundary",
+                "Content-Disposition: form-data; name=\"lorem\"; filename=\"lorem.txt\"",
+                "Content-Type: text/plain",
+                "Content-Length: 3418",
+                "",
+                try! String(contentsOf: url),
+
+                "--SuperAwesomeBoundary--",
             ].joined(separator: "\r\n")
         )
     }
-
-    static var allTests = [
-        ("testEncodeNothing", testEncodeNothing),
-        ("testEncodeText", testEncodeText),
-        ("testEncodeData", testEncodeData),
-        ("testEncodeEverything", testEncodeEverything),
-    ]
 }

Tests/OsirisTests/Resources/lorem.txt

@@ -0,0 +1,9 @@
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce vulputate massa eget turpis auctor, bibendum tincidunt purus tempus. Aliquam id risus quis est porttitor mollis a vitae mauris. Nam magna odio, egestas ut viverra vitae, pretium vel sem. Fusce imperdiet enim non orci placerat, quis eleifend nunc malesuada. Suspendisse scelerisque lacinia volutpat. Duis efficitur ut dolor at iaculis. Donec consequat eros nec augue pellentesque, ut dapibus mauris pharetra. Maecenas eget pharetra justo. Proin bibendum a lectus sed rutrum. In faucibus dolor commodo, ornare lacus ac, vulputate quam.
+
+Nam mollis tortor et elit aliquam suscipit. Maecenas eget erat et justo vehicula scelerisque ac quis erat. Nullam placerat felis sit amet nibh ullamcorper, at volutpat est congue. Fusce in pharetra eros. Phasellus laoreet arcu a libero imperdiet commodo. Suspendisse facilisis, libero in pellentesque aliquam, enim libero pulvinar nibh, ac aliquam nibh ante nec velit. Proin condimentum elit ex, ut dictum tellus volutpat ac. Nunc fermentum nisl id est gravida, nec pellentesque libero blandit. Etiam vitae pellentesque eros. Pellentesque consequat urna ligula, id imperdiet lorem efficitur non. Cras at placerat nulla, vel luctus odio. Phasellus a posuere velit. Ut in magna in augue sagittis elementum ac vitae turpis. Quisque ornare velit nec cursus dignissim.
+
+In hac habitasse platea dictumst. Morbi lectus leo, rhoncus non scelerisque imperdiet, porttitor non neque. In vehicula posuere convallis. Nullam eu varius nisi. Ut euismod egestas mi. Phasellus eget urna egestas, suscipit ex sed, ultricies leo. Suspendisse consequat in massa quis ullamcorper. Sed eleifend nulla tellus, vitae posuere augue vulputate ac. Curabitur iaculis vulputate dui ornare laoreet.
+
+Maecenas ac eros cursus, ultricies diam vitae, sollicitudin metus. Suspendisse quis mattis ex. Ut vel ullamcorper sapien. Sed in mauris sed ante imperdiet consectetur vitae sed dui. Aliquam in tempus mi. Phasellus vitae augue volutpat, sodales augue sit amet, pellentesque ante. Integer nec lacinia nunc. Morbi magna eros, placerat vitae eros sed, varius imperdiet sem. Vestibulum ultrices finibus ultrices. Cras condimentum vestibulum nisi mattis posuere. Sed sed nisl nec odio posuere ullamcorper sit amet sed quam. Ut id suscipit turpis. Aliquam et sem quis ligula bibendum aliquam. Praesent efficitur vel nibh in commodo. Donec porta, nunc at aliquam consectetur, ante diam eleifend ante, at elementum est neque in ante. Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+
+Donec tincidunt massa urna, in ultrices arcu vulputate in. Quisque placerat nisi ipsum, eget aliquet enim dictum vel. Ut sed nulla odio. Etiam in vulputate dui. Nunc elementum sodales neque, quis tristique massa feugiat eget. Donec ultrices erat facilisis ipsum volutpat tincidunt. Etiam efficitur luctus ipsum eu convallis. Ut elementum, nisl id aliquet ultrices, nunc quam gravida lectus, non ullamcorper elit justo non lorem. Sed tincidunt efficitur lectus ac molestie. Sed vel tellus eu tortor finibus fermentum a elementum mi. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus suscipit sed tortor tempor congue. Proin molestie bibendum metus. Nam ullamcorper, nunc placerat commodo egestas, ante odio faucibus nisi, vel vulputate augue nulla at ex. Duis scelerisque interdum felis, sed fringilla urna. Mauris lacinia felis gravida egestas dictum.