From f9c124e0a2e8b0c9b9a757266ad377f891eeb153 Mon Sep 17 00:00:00 2001 From: Itai Ferber Date: Wed, 16 Jan 2019 10:47:22 -0800 Subject: [PATCH] Merge pull request #21754 from itaiferber/data-inlinability-refinements (#21853) Refine Data inlinability (cherry picked from commit 6c576109e1983c91f6a14cd7d35056937fa5c9c6) --- stdlib/public/SDK/Foundation/Data.swift | 649 ++++++++++++------------ test/stdlib/TestData.swift | 52 +- 2 files changed, 365 insertions(+), 336 deletions(-) diff --git a/stdlib/public/SDK/Foundation/Data.swift b/stdlib/public/SDK/Foundation/Data.swift index 921f21db70f78..ad46097fb3a9c 100644 --- a/stdlib/public/SDK/Foundation/Data.swift +++ b/stdlib/public/SDK/Foundation/Data.swift @@ -17,7 +17,7 @@ import Darwin #elseif os(Linux) import Glibc -@inlinable +@inlinable // This is @inlinable as trivially computable. fileprivate func malloc_good_size(_ size: Int) -> Int { return size } @@ -62,14 +62,14 @@ internal func __NSDataIsCompact(_ data: NSData) -> Bool { #endif +// Underlying storage representation for medium and large data. +// Inlinability strategy: methods from here should not inline into InlineSlice or LargeSlice unless trivial. @usableFromInline internal final class _DataStorage { - @usableFromInline - static let maxSize = Int.max >> 1 - @usableFromInline - static let vmOpsThreshold = NSPageSize() * 4 + @usableFromInline static let maxSize = Int.max >> 1 + @usableFromInline static let vmOpsThreshold = NSPageSize() * 4 - @inlinable + @inlinable // This is @inlinable as trivially forwarding, and does not escape the _DataStorage boundary layer. static func allocate(_ size: Int, _ clear: Bool) -> UnsafeMutableRawPointer? { if clear { return calloc(1, size) @@ -78,7 +78,7 @@ internal final class _DataStorage { } } - @inlinable + @usableFromInline // This is not @inlinable as it is a non-trivial, non-generic function. static func move(_ dest_: UnsafeMutableRawPointer, _ source_: UnsafeRawPointer?, _ num_: Int) { var dest = dest_ var source = source_ @@ -95,50 +95,46 @@ internal final class _DataStorage { } } - @inlinable + @inlinable // This is @inlinable as trivially forwarding, and does not escape the _DataStorage boundary layer. static func shouldAllocateCleared(_ size: Int) -> Bool { return (size > (128 * 1024)) } - @usableFromInline - var _bytes: UnsafeMutableRawPointer? - @usableFromInline - var _length: Int - @usableFromInline - var _capacity: Int - @usableFromInline - var _needToZero: Bool - @usableFromInline - var _deallocator: ((UnsafeMutableRawPointer, Int) -> Void)? - @usableFromInline - var _offset: Int + @usableFromInline var _bytes: UnsafeMutableRawPointer? + @usableFromInline var _length: Int + @usableFromInline var _capacity: Int + @usableFromInline var _needToZero: Bool + @usableFromInline var _deallocator: ((UnsafeMutableRawPointer, Int) -> Void)? + @usableFromInline var _offset: Int - @inlinable + @inlinable // This is @inlinable as trivially computable. var bytes: UnsafeRawPointer? { return UnsafeRawPointer(_bytes)?.advanced(by: -_offset) } - @inlinable + @inlinable // This is @inlinable despite escaping the _DataStorage boundary layer because it is generic and trivially forwarding. @discardableResult func withUnsafeBytes(in range: Range, apply: (UnsafeRawBufferPointer) throws -> Result) rethrows -> Result { return try apply(UnsafeRawBufferPointer(start: _bytes?.advanced(by: range.lowerBound - _offset), count: Swift.min(range.upperBound - range.lowerBound, _length))) } - @inlinable + @inlinable // This is @inlinable despite escaping the _DataStorage boundary layer because it is generic and trivially forwarding. @discardableResult func withUnsafeMutableBytes(in range: Range, apply: (UnsafeMutableRawBufferPointer) throws -> Result) rethrows -> Result { return try apply(UnsafeMutableRawBufferPointer(start: _bytes!.advanced(by:range.lowerBound - _offset), count: Swift.min(range.upperBound - range.lowerBound, _length))) } - @inlinable + @inlinable // This is @inlinable as trivially computable. var mutableBytes: UnsafeMutableRawPointer? { return _bytes?.advanced(by: -_offset) } - @inlinable - var capacity: Int { return _capacity } + @inlinable // This is @inlinable as trivially computable. + var capacity: Int { + return _capacity + } - @inlinable + @inlinable // This is @inlinable as trivially computable. var length: Int { get { return _length @@ -148,14 +144,14 @@ internal final class _DataStorage { } } - @inlinable + @inlinable // This is inlinable as trivially computable. var isExternallyOwned: Bool { // all _DataStorages will have some sort of capacity, because empty cases hit the .empty enum _Representation // anything with 0 capacity means that we have not allocated this pointer and concequently mutation is not ours to make. return _capacity == 0 } - @inlinable + @usableFromInline // This is not @inlinable as it is a non-trivial, non-generic function. func ensureUniqueBufferReference(growingTo newLength: Int = 0, clear: Bool = false) { guard isExternallyOwned || newLength > _capacity else { return } @@ -256,7 +252,7 @@ internal final class _DataStorage { } } - @inlinable + @inlinable // This is @inlinable as it does not escape the _DataStorage boundary layer. func _freeBytes() { if let bytes = _bytes { if let dealloc = _deallocator { @@ -268,13 +264,13 @@ internal final class _DataStorage { _deallocator = nil } - @usableFromInline + @inlinable // This is @inlinable despite escaping the _DataStorage boundary layer because it is trivially computed. func enumerateBytes(in range: Range, _ block: (_ buffer: UnsafeBufferPointer, _ byteIndex: Data.Index, _ stop: inout Bool) -> Void) { var stopv: Bool = false block(UnsafeBufferPointer(start: _bytes?.advanced(by: range.lowerBound - _offset).assumingMemoryBound(to: UInt8.self), count: Swift.min(range.upperBound - range.lowerBound, _length)), 0, &stopv) } - @inlinable + @inlinable // This is @inlinable as it does not escape the _DataStorage boundary layer. func setLength(_ length: Int) { let origLength = _length let newLength = length @@ -288,7 +284,7 @@ internal final class _DataStorage { _length = newLength } - @inlinable + @inlinable // This is @inlinable as it does not escape the _DataStorage boundary layer. func append(_ bytes: UnsafeRawPointer, length: Int) { precondition(length >= 0, "Length of appending bytes must not be negative") let origLength = _length @@ -300,72 +296,24 @@ internal final class _DataStorage { _DataStorage.move(_bytes!.advanced(by: origLength), bytes, length) } - // fast-path for appending directly from another data storage - @inlinable - func append(_ otherData: _DataStorage, startingAt start: Int, endingAt end: Int) { - let otherLength = otherData.length - if otherLength == 0 { return } - if let bytes = otherData.bytes { - append(bytes.advanced(by: start), length: end - start) - } - } - - @inlinable - func append(_ otherData: Data) { - guard otherData.count > 0 else { return } - otherData.withUnsafeBytes { - append($0.baseAddress!, length: $0.count) - } - } - - @inlinable - func increaseLength(by extraLength: Int) { - if extraLength == 0 { return } - - let origLength = _length - let newLength = origLength + extraLength - if _capacity < newLength || _bytes == nil { - ensureUniqueBufferReference(growingTo: newLength, clear: true) - } else if _needToZero { - memset(_bytes!.advanced(by: origLength), 0, extraLength) - } - _length = newLength - } - - @inlinable + @inlinable // This is @inlinable despite escaping the _DataStorage boundary layer because it is trivially computed. func get(_ index: Int) -> UInt8 { return _bytes!.advanced(by: index - _offset).assumingMemoryBound(to: UInt8.self).pointee } - @inlinable + @inlinable // This is @inlinable despite escaping the _DataStorage boundary layer because it is trivially computed. func set(_ index: Int, to value: UInt8) { ensureUniqueBufferReference() _bytes!.advanced(by: index - _offset).assumingMemoryBound(to: UInt8.self).pointee = value } - @inlinable + @inlinable // This is @inlinable despite escaping the _DataStorage boundary layer because it is trivially computed. func copyBytes(to pointer: UnsafeMutableRawPointer, from range: Range) { let offsetPointer = UnsafeRawBufferPointer(start: _bytes?.advanced(by: range.lowerBound - _offset), count: Swift.min(range.upperBound - range.lowerBound, _length)) UnsafeMutableRawBufferPointer(start: pointer, count: range.upperBound - range.lowerBound).copyMemory(from: offsetPointer) } - @inlinable - func replaceBytes(in range: NSRange, with bytes: UnsafeRawPointer?) { - if range.length == 0 { return } - if _length < range.location + range.length { - let newLength = range.location + range.length - if _capacity < newLength { - ensureUniqueBufferReference(growingTo: newLength, clear: false) - } - _length = newLength - } else { - ensureUniqueBufferReference() - } - _DataStorage.move(_bytes!.advanced(by: range.location - _offset), bytes!, range.length) - - } - - @inlinable + @usableFromInline // This is not @inlinable as it is a non-trivial, non-generic function. func replaceBytes(in range_: NSRange, with replacementBytes: UnsafeRawPointer?, length replacementLength: Int) { let range = NSRange(location: range_.location - _offset, length: range_.length) let currentLength = _length @@ -398,7 +346,7 @@ internal final class _DataStorage { } } - @inlinable + @usableFromInline // This is not @inlinable as it is a non-trivial, non-generic function. func resetBytes(in range_: Range) { let range = NSRange(location: range_.lowerBound - _offset, length: range_.upperBound - range_.lowerBound) if range.length == 0 { return } @@ -414,12 +362,7 @@ internal final class _DataStorage { memset(_bytes!.advanced(by: range.location), 0, range.length) } - @inlinable - convenience init() { - self.init(capacity: 0) - } - - @usableFromInline + @usableFromInline // This is not @inlinable as a non-trivial, non-convenience initializer. init(length: Int) { precondition(length < _DataStorage.maxSize) var capacity = (length < 1024 * 1024 * 1024) ? length + (length >> 2) : length @@ -436,8 +379,8 @@ internal final class _DataStorage { setLength(length) } - @usableFromInline - init(capacity capacity_: Int) { + @usableFromInline // This is not @inlinable as a non-convience initializer. + init(capacity capacity_: Int = 0) { var capacity = capacity_ precondition(capacity < _DataStorage.maxSize) if _DataStorage.vmOpsThreshold <= capacity { @@ -450,7 +393,7 @@ internal final class _DataStorage { _offset = 0 } - @usableFromInline + @usableFromInline // This is not @inlinable as a non-convience initializer. init(bytes: UnsafeRawPointer?, length: Int) { precondition(length < _DataStorage.maxSize) _offset = 0 @@ -478,7 +421,7 @@ internal final class _DataStorage { } } - @usableFromInline + @usableFromInline // This is not @inlinable as a non-convience initializer. init(bytes: UnsafeMutableRawPointer?, length: Int, copy: Bool, deallocator: ((UnsafeMutableRawPointer, Int) -> Void)?, offset: Int) { precondition(length < _DataStorage.maxSize) _offset = offset @@ -522,7 +465,7 @@ internal final class _DataStorage { } } - @usableFromInline + @usableFromInline // This is not @inlinable as a non-convience initializer. init(immutableReference: NSData, offset: Int) { _offset = offset _bytes = UnsafeMutableRawPointer(mutating: immutableReference.bytes) @@ -534,7 +477,7 @@ internal final class _DataStorage { } } - @usableFromInline + @usableFromInline // This is not @inlinable as a non-convience initializer. init(mutableReference: NSMutableData, offset: Int) { _offset = offset _bytes = mutableReference.mutableBytes @@ -546,7 +489,7 @@ internal final class _DataStorage { } } - @usableFromInline + @usableFromInline // This is not @inlinable as a non-convience initializer. init(customReference: NSData, offset: Int) { _offset = offset _bytes = UnsafeMutableRawPointer(mutating: customReference.bytes) @@ -558,7 +501,7 @@ internal final class _DataStorage { } } - @usableFromInline + @usableFromInline // This is not @inlinable as a non-convience initializer. init(customMutableReference: NSMutableData, offset: Int) { _offset = offset _bytes = customMutableReference.mutableBytes @@ -574,12 +517,12 @@ internal final class _DataStorage { _freeBytes() } - @inlinable + @inlinable // This is @inlinable despite escaping the _DataStorage boundary layer because it is trivially computed. func mutableCopy(_ range: Range) -> _DataStorage { return _DataStorage(bytes: _bytes?.advanced(by: range.lowerBound - _offset), length: range.upperBound - range.lowerBound, copy: true, deallocator: nil, offset: range.lowerBound) } - @inlinable + @inlinable // This is @inlinable despite escaping the _DataStorage boundary layer because it is generic and trivially computed. func withInteriorPointerReference(_ range: Range, _ work: (NSData) throws -> T) rethrows -> T { if range.isEmpty { return try work(NSData()) // zero length data can be optimized as a singleton @@ -587,10 +530,8 @@ internal final class _DataStorage { return try work(NSData(bytesNoCopy: _bytes!.advanced(by: range.lowerBound - _offset), length: range.upperBound - range.lowerBound, freeWhenDone: false)) } - // This is used to create bridged Datas into Objective-C contexts, the class name is private and should not be emitted into clients. - // Consequently this should never be inlined. + @inline(never) // This is not @inlinable to avoid emission of the private `__NSSwiftData` class name into clients. @usableFromInline - @inline(never) func bridgedReference(_ range: Range) -> NSData { if range.isEmpty { return NSData() // zero length data can be optimized as a singleton @@ -598,11 +539,6 @@ internal final class _DataStorage { return __NSSwiftData(backing: self, range: range) } - - @inlinable - func subdata(in range: Range) -> Data { - return Data(bytes: _bytes!.advanced(by: range.lowerBound - _offset), count: range.upperBound - range.lowerBound) - } } // NOTE: older runtimes called this _NSSwiftData. The two must @@ -676,36 +612,28 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl public typealias Index = Int public typealias Indices = Range + // A small inline buffer of bytes suitable for stack-allocation of small data. + // Inlinability strategy: everything here should be inlined for direct operation on the stack wherever possible. @usableFromInline @_fixed_layout internal struct InlineData { #if arch(x86_64) || arch(arm64) || arch(s390x) || arch(powerpc64) || arch(powerpc64le) - @usableFromInline - typealias Buffer = (UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, - UInt8, UInt8, UInt8, UInt8, UInt8, UInt8) //len //enum - @usableFromInline - var bytes: Buffer + @usableFromInline typealias Buffer = (UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, + UInt8, UInt8, UInt8, UInt8, UInt8, UInt8) //len //enum + @usableFromInline var bytes: Buffer #elseif arch(i386) || arch(arm) - @usableFromInline - typealias Buffer = (UInt8, UInt8, UInt8, UInt8, - UInt8, UInt8) //len //enum - @usableFromInline - var bytes: Buffer + @usableFromInline typealias Buffer = (UInt8, UInt8, UInt8, UInt8, + UInt8, UInt8) //len //enum + @usableFromInline var bytes: Buffer #endif - @usableFromInline - var length: UInt8 + @usableFromInline var length: UInt8 - @inlinable + @inlinable // This is @inlinable as trivially computable. static func canStore(count: Int) -> Bool { return count <= MemoryLayout.size } - @inlinable - init() { - self.init(count: 0) - } - - @inlinable + @inlinable // This is @inlinable as a convenience initializer. init(_ srcBuffer: UnsafeRawBufferPointer) { self.init(count: srcBuffer.count) if srcBuffer.count > 0 { @@ -715,19 +643,18 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable - init(count: Int) { + @inlinable // This is @inlinable as a trivial initializer. + init(count: Int = 0) { assert(count <= MemoryLayout.size) #if arch(x86_64) || arch(arm64) || arch(s390x) || arch(powerpc64) || arch(powerpc64le) bytes = (UInt8(0), UInt8(0), UInt8(0), UInt8(0), UInt8(0), UInt8(0), UInt8(0), UInt8(0), UInt8(0), UInt8(0), UInt8(0), UInt8(0), UInt8(0), UInt8(0)) #elseif arch(i386) || arch(arm) - bytes = (UInt8(0), UInt8(0), UInt8(0), UInt8(0), - UInt8(0), UInt8(0)) + bytes = (UInt8(0), UInt8(0), UInt8(0), UInt8(0), UInt8(0), UInt8(0)) #endif length = UInt8(count) } - @inlinable + @inlinable // This is @inlinable as a convenience initializer. init(_ slice: InlineSlice, count: Int) { self.init(count: count) Swift.withUnsafeMutableBytes(of: &bytes) { dstBuffer in @@ -737,7 +664,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as a convenience initializer. init(_ slice: LargeSlice, count: Int) { self.init(count: count) Swift.withUnsafeMutableBytes(of: &bytes) { dstBuffer in @@ -747,12 +674,12 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as trivially computable. var capacity: Int { return MemoryLayout.size } - @inlinable + @inlinable // This is @inlinable as trivially computable. var count: Int { get { return Int(length) @@ -763,13 +690,17 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable - var startIndex: Int { return 0 } + @inlinable // This is @inlinable as trivially computable. + var startIndex: Int { + return 0 + } - @inlinable - var endIndex: Int { return count } + @inlinable // This is @inlinable as trivially computable. + var endIndex: Int { + return count + } - @inlinable + @inlinable // This is @inlinable as a generic, trivially forwarding function. func withUnsafeBytes(_ apply: (UnsafeRawBufferPointer) throws -> Result) rethrows -> Result { let count = Int(length) return try Swift.withUnsafeBytes(of: bytes) { (rawBuffer) throws -> Result in @@ -777,7 +708,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as a generic, trivially forwarding function. mutating func withUnsafeMutableBytes(_ apply: (UnsafeMutableRawBufferPointer) throws -> Result) rethrows -> Result { let count = Int(length) return try Swift.withUnsafeMutableBytes(of: &bytes) { (rawBuffer) throws -> Result in @@ -785,7 +716,15 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as tribially computable. + mutating func append(byte: UInt8) { + let count = self.count + assert(count + 1 <= MemoryLayout.size) + Swift.withUnsafeMutableBytes(of: &bytes) { $0[count] = byte } + self.length += 1 + } + + @inlinable // This is @inlinable as trivially computable. mutating func append(contentsOf buffer: UnsafeRawBufferPointer) { guard buffer.count > 0 else { return } assert(count + buffer.count <= MemoryLayout.size) @@ -793,11 +732,11 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl _ = Swift.withUnsafeMutableBytes(of: &bytes) { rawBuffer in rawBuffer.baseAddress?.advanced(by: cnt).copyMemory(from: buffer.baseAddress!, byteCount: buffer.count) } - length += UInt8(buffer.count) + length += UInt8(buffer.count) } - @inlinable + @inlinable // This is @inlinable as trivially computable. subscript(index: Index) -> UInt8 { get { assert(index <= MemoryLayout.size) @@ -815,7 +754,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as trivially computable. mutating func resetBytes(in range: Range) { assert(range.lowerBound <= MemoryLayout.size) assert(range.upperBound <= MemoryLayout.size) @@ -823,12 +762,13 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl if count < range.upperBound { count = range.upperBound } + let _ = Swift.withUnsafeMutableBytes(of: &bytes) { rawBuffer in memset(rawBuffer.baseAddress?.advanced(by: range.lowerBound), 0, range.upperBound - range.lowerBound) } } - @inlinable + @usableFromInline // This is not @inlinable as it is a non-trivial, non-generic function. mutating func replaceSubrange(_ subrange: Range, with replacementBytes: UnsafeRawPointer?, count replacementLength: Int) { assert(subrange.lowerBound <= MemoryLayout.size) assert(subrange.upperBound <= MemoryLayout.size) @@ -852,7 +792,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl count = resultingLength } - @inlinable + @inlinable // This is @inlinable as trivially computable. func copyBytes(to pointer: UnsafeMutableRawPointer, from range: Range) { precondition(startIndex <= range.lowerBound, "index \(range.lowerBound) is out of bounds of \(startIndex)..<\(endIndex)") precondition(range.lowerBound <= endIndex, "index \(range.lowerBound) is out of bounds of \(startIndex)..<\(endIndex)") @@ -866,91 +806,101 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable - var hashValue: Int { - let count = Int(length) - return Swift.withUnsafeBytes(of: bytes) { (rawBuffer) -> Int in - return Int(bitPattern: CFHashBytes(UnsafeMutablePointer(mutating: rawBuffer.baseAddress?.assumingMemoryBound(to: UInt8.self)), count)) + @inline(__always) // This should always be inlined into _Representation.hash(into:). + func hash(into hasher: inout Hasher) { + // **NOTE**: this uses `count` (an Int) and NOT `length` (a UInt8) + // Despite having the same value, they hash differently. InlineSlice and LargeSlice both use `count` (an Int); if you combine the same bytes but with `length` over `count`, you can get a different hash. + // + // This affects slices, which are InlineSlice and not InlineData: + // + // let d = Data([0xFF, 0xFF]) // InlineData + // let s = Data([0, 0xFF, 0xFF]).dropFirst() // InlineSlice + // assert(s == d) + // assert(s.hashValue == d.hashValue) + hasher.combine(count) + + Swift.withUnsafeBytes(of: bytes) { + // We have access to the full byte buffer here, but not all of it is meaningfully used (bytes past self.length may be garbage). + let bytes = UnsafeRawBufferPointer(start: $0.baseAddress, count: self.count) + hasher.combine(bytes: bytes) } } } #if arch(x86_64) || arch(arm64) || arch(s390x) || arch(powerpc64) || arch(powerpc64le) - @usableFromInline - internal typealias HalfInt = Int32 + @usableFromInline internal typealias HalfInt = Int32 #elseif arch(i386) || arch(arm) - @usableFromInline - internal typealias HalfInt = Int16 + @usableFromInline internal typealias HalfInt = Int16 #endif + // A buffer of bytes too large to fit in an InlineData, but still small enough to fit a storage pointer + range in two words. + // Inlinability strategy: everything here should be easily inlinable as large _DataStorage methods should not inline into here. @usableFromInline @_fixed_layout internal struct InlineSlice { // ***WARNING*** // These ivars are specifically laid out so that they cause the enum _Representation to be 16 bytes on 64 bit platforms. This means we _MUST_ have the class type thing last - @usableFromInline - var slice: Range - @usableFromInline - var storage: _DataStorage + @usableFromInline var slice: Range + @usableFromInline var storage: _DataStorage - @inlinable + @inlinable // This is @inlinable as trivially computable. static func canStore(count: Int) -> Bool { return count < HalfInt.max } - @inlinable + @inlinable // This is @inlinable as a convenience initializer. init(_ buffer: UnsafeRawBufferPointer) { assert(buffer.count < HalfInt.max) self.init(_DataStorage(bytes: buffer.baseAddress, length: buffer.count), count: buffer.count) } - @inlinable + @inlinable // This is @inlinable as a convenience initializer. init(capacity: Int) { assert(capacity < HalfInt.max) self.init(_DataStorage(capacity: capacity), count: 0) } - @inlinable + @inlinable // This is @inlinable as a convenience initializer. init(count: Int) { assert(count < HalfInt.max) self.init(_DataStorage(length: count), count: count) } - @inlinable + @inlinable // This is @inlinable as a convenience initializer. init(_ inline: InlineData) { assert(inline.count < HalfInt.max) self.init(inline.withUnsafeBytes { return _DataStorage(bytes: $0.baseAddress, length: $0.count) }, count: inline.count) } - @inlinable + @inlinable // This is @inlinable as a convenience initializer. init(_ inline: InlineData, range: Range) { assert(range.lowerBound < HalfInt.max) assert(range.upperBound < HalfInt.max) self.init(inline.withUnsafeBytes { return _DataStorage(bytes: $0.baseAddress, length: $0.count) }, range: range) } - @inlinable + @inlinable // This is @inlinable as a convenience initializer. init(_ large: LargeSlice) { assert(large.range.lowerBound < HalfInt.max) assert(large.range.upperBound < HalfInt.max) self.init(large.storage, range: large.range) } - @inlinable + @inlinable // This is @inlinable as a convenience initializer. init(_ large: LargeSlice, range: Range) { assert(range.lowerBound < HalfInt.max) assert(range.upperBound < HalfInt.max) self.init(large.storage, range: range) } - @inlinable + @inlinable // This is @inlinable as a trivial initializer. init(_ storage: _DataStorage, count: Int) { assert(count < HalfInt.max) self.storage = storage slice = 0..) { assert(range.lowerBound < HalfInt.max) assert(range.upperBound < HalfInt.max) @@ -958,31 +908,36 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl slice = HalfInt(range.lowerBound).. { get { return Int(slice.lowerBound)..(_ apply: (UnsafeRawBufferPointer) throws -> Result) rethrows -> Result { return try storage.withUnsafeBytes(in: range, apply: apply) } - @inlinable + @inlinable // This is @inlinable as a generic, trivially forwarding function. mutating func withUnsafeMutableBytes(_ apply: (UnsafeMutableRawBufferPointer) throws -> Result) rethrows -> Result { ensureUniqueReference() return try storage.withUnsafeMutableBytes(in: range, apply: apply) } - @inlinable + @inlinable // This is @inlinable as reasonably small. mutating func append(contentsOf buffer: UnsafeRawBufferPointer) { assert(endIndex + buffer.count < HalfInt.max) ensureUniqueReference() @@ -1026,7 +981,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl slice = slice.lowerBound.. UInt8 { get { assert(index < HalfInt.max) @@ -1043,12 +998,12 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as trivially forwarding. func bridgedReference() -> NSData { return storage.bridgedReference(self.range) } - @inlinable + @inlinable // This is @inlinable as reasonably small. mutating func resetBytes(in range: Range) { assert(range.lowerBound < HalfInt.max) assert(range.upperBound < HalfInt.max) @@ -1060,7 +1015,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as reasonably small. mutating func replaceSubrange(_ subrange: Range, with bytes: UnsafeRawPointer?, count cnt: Int) { precondition(startIndex <= subrange.lowerBound, "index \(subrange.lowerBound) is out of bounds of \(startIndex)..<\(endIndex)") precondition(subrange.lowerBound <= endIndex, "index \(subrange.lowerBound) is out of bounds of \(startIndex)..<\(endIndex)") @@ -1075,7 +1030,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl slice = slice.lowerBound..) { precondition(startIndex <= range.lowerBound, "index \(range.lowerBound) is out of bounds of \(startIndex)..<\(endIndex)") precondition(range.lowerBound <= endIndex, "index \(range.lowerBound) is out of bounds of \(startIndex)..<\(endIndex)") @@ -1084,79 +1039,90 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl storage.copyBytes(to: pointer, from: range) } - @inlinable - var hashValue: Int { - let hashRange = startIndex.. for when the range of a data buffer is too large to whole in a single word. + // Inlinability strategy: everything should be inlinable as trivial. @usableFromInline @_fixed_layout internal final class RangeReference { - @usableFromInline - var range: Range + @usableFromInline var range: Range - @inlinable - var lowerBound: Int { return range.lowerBound } + @inlinable @inline(__always) // This is @inlinable as trivially forwarding. + var lowerBound: Int { + return range.lowerBound + } - @inlinable - var upperBound: Int { return range.upperBound } + @inlinable @inline(__always) // This is @inlinable as trivially forwarding. + var upperBound: Int { + return range.upperBound + } - @inlinable - var count: Int { return range.upperBound - range.lowerBound } + @inlinable @inline(__always) // This is @inlinable as trivially computable. + var count: Int { + return range.upperBound - range.lowerBound + } - @inlinable + @inlinable @inline(__always) // This is @inlinable as a trivial initializer. init(_ range: Range) { self.range = range } } + // A buffer of bytes whose range is too large to fit in a signle word. Used alongside a RangeReference to make it fit into _Representation's two-word size. + // Inlinability strategy: everything here should be easily inlinable as large _DataStorage methods should not inline into here. @usableFromInline @_fixed_layout internal struct LargeSlice { // ***WARNING*** // These ivars are specifically laid out so that they cause the enum _Representation to be 16 bytes on 64 bit platforms. This means we _MUST_ have the class type thing last - @usableFromInline - var slice: RangeReference - @usableFromInline - var storage: _DataStorage + @usableFromInline var slice: RangeReference + @usableFromInline var storage: _DataStorage - @inlinable + @inlinable // This is @inlinable as a convenience initializer. init(_ buffer: UnsafeRawBufferPointer) { self.init(_DataStorage(bytes: buffer.baseAddress, length: buffer.count), count: buffer.count) } - @inlinable + @inlinable // This is @inlinable as a convenience initializer. init(capacity: Int) { self.init(_DataStorage(capacity: capacity), count: 0) } - @inlinable + @inlinable // This is @inlinable as a convenience initializer. init(count: Int) { self.init(_DataStorage(length: count), count: count) } - @inlinable + @inlinable // This is @inlinable as a convenience initializer. init(_ inline: InlineData) { - self.init(inline.withUnsafeBytes { return _DataStorage(bytes: $0.baseAddress, length: $0.count) }, count: inline.count) + let storage = inline.withUnsafeBytes { return _DataStorage(bytes: $0.baseAddress, length: $0.count) } + self.init(storage, count: inline.count) } - @inlinable + @inlinable // This is @inlinable as a trivial initializer. init(_ slice: InlineSlice) { self.storage = slice.storage self.slice = RangeReference(slice.range) } - @inlinable + @inlinable // This is @inlinable as a trivial initializer. init(_ storage: _DataStorage, count: Int) { self.storage = storage - slice = RangeReference(0.. { return slice.range } + @inlinable // This is @inlinable as it is trivially forwarding. + var range: Range { + return slice.range + } - @inlinable + @inlinable // This is @inlinable as a generic, trivially forwarding function. func withUnsafeBytes(_ apply: (UnsafeRawBufferPointer) throws -> Result) rethrows -> Result { return try storage.withUnsafeBytes(in: range, apply: apply) } - @inlinable + @inlinable // This is @inlinable as a generic, trivially forwarding function. mutating func withUnsafeMutableBytes(_ apply: (UnsafeMutableRawBufferPointer) throws -> Result) rethrows -> Result { ensureUniqueReference() return try storage.withUnsafeMutableBytes(in: range, apply: apply) } - @inlinable + @inlinable // This is @inlinable as reasonably small. mutating func append(contentsOf buffer: UnsafeRawBufferPointer) { ensureUniqueReference() storage.replaceBytes(in: NSRange(location: range.upperBound, length: storage.length - (range.upperBound - storage._offset)), with: buffer.baseAddress, length: buffer.count) slice.range = slice.range.lowerBound.. UInt8 { get { precondition(startIndex <= index, "index \(index) is out of bounds of \(startIndex)..<\(endIndex)") @@ -1232,12 +1204,12 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as trivially forwarding. func bridgedReference() -> NSData { return storage.bridgedReference(self.range) } - @inlinable + @inlinable // This is @inlinable as reasonably small. mutating func resetBytes(in range: Range) { precondition(range.lowerBound <= endIndex, "index \(range.lowerBound) is out of bounds of \(startIndex)..<\(endIndex)") ensureUniqueReference() @@ -1247,7 +1219,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as reasonably small. mutating func replaceSubrange(_ subrange: Range, with bytes: UnsafeRawPointer?, count cnt: Int) { precondition(startIndex <= subrange.lowerBound, "index \(subrange.lowerBound) is out of bounds of \(startIndex)..<\(endIndex)") precondition(subrange.lowerBound <= endIndex, "index \(subrange.lowerBound) is out of bounds of \(startIndex)..<\(endIndex)") @@ -1262,7 +1234,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl slice.range = slice.range.lowerBound..) { precondition(startIndex <= range.lowerBound, "index \(range.lowerBound) is out of bounds of \(startIndex)..<\(endIndex)") precondition(range.lowerBound <= endIndex, "index \(range.lowerBound) is out of bounds of \(startIndex)..<\(endIndex)") @@ -1271,15 +1243,20 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl storage.copyBytes(to: pointer, from: range) } - @inlinable - var hashValue: Int { - let hashRange = startIndex.. 0 else { return } switch self { @@ -1402,7 +1379,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as reasonably small. var count: Int { get { switch self { @@ -1413,6 +1390,8 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } set(newValue) { + // HACK: The definition of this inline function takes an inout reference to self, giving the optimizer a unique referencing guarantee. + // This allows us to avoid excessive retain-release traffic around modifying enum values, and inlining the function then avoids the additional frame. @inline(__always) func apply(_ representation: inout _Representation, _ newValue: Int) -> _Representation? { switch representation { @@ -1477,7 +1456,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as a generic, trivially forwarding function. func withUnsafeBytes(_ apply: (UnsafeRawBufferPointer) throws -> Result) rethrows -> Result { switch self { case .empty: @@ -1492,7 +1471,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as a generic, trivially forwarding function. mutating func withUnsafeMutableBytes(_ apply: (UnsafeMutableRawBufferPointer) throws -> Result) rethrows -> Result { switch self { case .empty: @@ -1512,7 +1491,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as a generic, trivially forwarding function. func withInteriorPointerReference(_ work: (NSData) throws -> T) rethrows -> T { switch self { case .empty: @@ -1528,7 +1507,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @usableFromInline // This is not @inlinable as it is a non-trivial, non-generic function. func enumerateBytes(_ block: (_ buffer: UnsafeBufferPointer, _ byteIndex: Index, _ stop: inout Bool) -> Void) { switch self { case .empty: @@ -1546,7 +1525,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as reasonably small. mutating func append(contentsOf buffer: UnsafeRawBufferPointer) { switch self { case .empty: @@ -1582,7 +1561,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as reasonably small. mutating func resetBytes(in range: Range) { switch self { case .empty: @@ -1600,7 +1579,6 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } break case .inline(var inline): - if inline.count < range.upperBound { if InlineSlice.canStore(count: range.upperBound) { var slice = InlineSlice(inline) @@ -1635,7 +1613,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @usableFromInline // This is not @inlinable as it is a non-trivial, non-generic function. mutating func replaceSubrange(_ subrange: Range, with bytes: UnsafeRawPointer?, count cnt: Int) { switch self { case .empty: @@ -1718,7 +1696,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as trivially forwarding. subscript(index: Index) -> UInt8 { get { switch self { @@ -1746,7 +1724,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as reasonably small. subscript(bounds: Range) -> Data { get { switch self { @@ -1796,7 +1774,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as trivially forwarding. var startIndex: Int { switch self { case .empty: return 0 @@ -1806,7 +1784,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as trivially forwarding. var endIndex: Int { switch self { case .empty: return 0 @@ -1816,7 +1794,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as trivially forwarding. func bridgedReference() -> NSData { switch self { case .empty: return NSData() @@ -1831,7 +1809,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as trivially forwarding. func copyBytes(to pointer: UnsafeMutableRawPointer, from range: Range) { switch self { case .empty: @@ -1847,17 +1825,17 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable - var hashValue: Int { + @inline(__always) // This should always be inlined into Data.hash(into:). + func hash(into hasher: inout Hasher) { switch self { case .empty: - return Int(bitPattern: CFHashBytes(nil, 0)) + hasher.combine(0) case .inline(let inline): - return inline.hashValue + inline.hash(into: &hasher) case .slice(let slice): - return slice.hashValue - case .large(let slice): - return slice.hashValue + slice.hash(into: &hasher) + case .large(let large): + large.hash(into: &hasher) } } } @@ -1885,8 +1863,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// A custom deallocator. case custom((UnsafeMutableRawPointer, Int) -> Void) - @usableFromInline - internal var _deallocator : ((UnsafeMutableRawPointer, Int) -> Void) { + @usableFromInline internal var _deallocator : ((UnsafeMutableRawPointer, Int) -> Void) { #if DEPLOYMENT_RUNTIME_SWIFT switch self { case .unmap: @@ -1922,7 +1899,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// /// - parameter bytes: A pointer to the memory. It will be copied. /// - parameter count: The number of bytes to copy. - @inlinable + @inlinable // This is @inlinable as a trivial initializer. public init(bytes: UnsafeRawPointer, count: Int) { _representation = _Representation(UnsafeRawBufferPointer(start: bytes, count: count)) } @@ -1930,7 +1907,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// Initialize a `Data` with copied memory content. /// /// - parameter buffer: A buffer pointer to copy. The size is calculated from `SourceType` and `buffer.count`. - @inlinable + @inlinable // This is @inlinable as a trivial, generic initializer. public init(buffer: UnsafeBufferPointer) { _representation = _Representation(UnsafeRawBufferPointer(buffer)) } @@ -1938,7 +1915,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// Initialize a `Data` with copied memory content. /// /// - parameter buffer: A buffer pointer to copy. The size is calculated from `SourceType` and `buffer.count`. - @inlinable + @inlinable // This is @inlinable as a trivial, generic initializer. public init(buffer: UnsafeMutableBufferPointer) { _representation = _Representation(UnsafeRawBufferPointer(buffer)) } @@ -1947,7 +1924,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// /// - parameter repeatedValue: A byte to initialize the pattern /// - parameter count: The number of bytes the data initially contains initialized to the repeatedValue - @inlinable + @inlinable // This is @inlinable as a convenience initializer. public init(repeating repeatedValue: UInt8, count: Int) { self.init(count: count) withUnsafeMutableBytes { (buffer: UnsafeMutableRawBufferPointer) -> Void in @@ -1964,7 +1941,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// If the capacity specified in `capacity` is greater than four memory pages in size, this may round the amount of requested memory up to the nearest full page. /// /// - parameter capacity: The size of the data. - @inlinable + @inlinable // This is @inlinable as a trivial initializer. public init(capacity: Int) { _representation = _Representation(capacity: capacity) } @@ -1972,13 +1949,13 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// Initialize a `Data` with the specified count of zeroed bytes. /// /// - parameter count: The number of bytes the data initially contains. - @inlinable + @inlinable // This is @inlinable as a trivial initializer. public init(count: Int) { _representation = _Representation(count: count) } /// Initialize an empty `Data`. - @inlinable + @inlinable // This is @inlinable as a trivial initializer. public init() { _representation = .empty } @@ -1990,7 +1967,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// - parameter bytes: A pointer to the bytes. /// - parameter count: The size of the bytes. /// - parameter deallocator: Specifies the mechanism to free the indicated buffer, or `.none`. - @inlinable + @inlinable // This is @inlinable as a trivial initializer. public init(bytesNoCopy bytes: UnsafeMutableRawPointer, count: Int, deallocator: Deallocator) { let whichDeallocator = deallocator._deallocator if count == 0 { @@ -2006,7 +1983,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// - parameter url: The `URL` to read. /// - parameter options: Options for the read operation. Default value is `[]`. /// - throws: An error in the Cocoa domain, if `url` cannot be read. - @inlinable + @inlinable // This is @inlinable as a convenience initializer. public init(contentsOf url: __shared URL, options: Data.ReadingOptions = []) throws { let d = try NSData(contentsOf: url, options: ReadingOptions(rawValue: options.rawValue)) self.init(bytes: d.bytes, count: d.length) @@ -2017,7 +1994,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// Returns nil when the input is not recognized as valid Base-64. /// - parameter base64String: The string to parse. /// - parameter options: Encoding options. Default value is `[]`. - @inlinable + @inlinable // This is @inlinable as a convenience initializer. public init?(base64Encoded base64String: __shared String, options: Data.Base64DecodingOptions = []) { if let d = NSData(base64Encoded: base64String, options: Base64DecodingOptions(rawValue: options.rawValue)) { self.init(bytes: d.bytes, count: d.length) @@ -2032,7 +2009,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// /// - parameter base64Data: Base-64, UTF-8 encoded input data. /// - parameter options: Decoding options. Default value is `[]`. - @inlinable + @inlinable // This is @inlinable as a convenience initializer. public init?(base64Encoded base64Data: __shared Data, options: Data.Base64DecodingOptions = []) { if let d = NSData(base64Encoded: base64Data, options: Base64DecodingOptions(rawValue: options.rawValue)) { self.init(bytes: d.bytes, count: d.length) @@ -2069,8 +2046,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } // slightly faster paths for common sequences - @inlinable - @inline(__always) + @inlinable // This is @inlinable as an important generic funnel point, despite being a non-trivial initializer. public init(_ elements: S) where S.Element == UInt8 { // If the sequence is already contiguous, access the underlying raw memory directly. if let contiguous = elements as? ContiguousBytes { @@ -2102,12 +2078,22 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl // Copy the contents of buffer... _representation = _Representation(UnsafeRawBufferPointer(start: base, count: endIndex)) - // ... and append the rest byte-wise. + // ... and append the rest byte-wise, buffering through an InlineData. + var buffer = InlineData() while let element = iter.next() { - Swift.withUnsafeBytes(of: element) { - _representation.append(contentsOf: $0) + if buffer.count < buffer.capacity { + buffer.append(byte: element) + } else { + buffer.withUnsafeBytes { _representation.append(contentsOf: $0) } + buffer.count = 0 } } + + // If we've still got bytes left in the buffer (i.e. the loop ended before we filled up the buffer and cleared it out), append them. + if buffer.count > 0 { + buffer.withUnsafeBytes { _representation.append(contentsOf: $0) } + buffer.count = 0 + } } } } @@ -2128,7 +2114,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl self.init(bytes) } - @inlinable + @inlinable // This is @inlinable as a trivial initializer. internal init(representation: _Representation) { _representation = representation } @@ -2136,13 +2122,13 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl // ----------------------------------- // MARK: - Properties and Functions - @inlinable + @inlinable // This is @inlinable as trivially forwarding. public mutating func reserveCapacity(_ minimumCapacity: Int) { _representation.reserveCapacity(minimumCapacity) } /// The number of bytes in the data. - @inlinable + @inlinable // This is @inlinable as trivially forwarding. public var count: Int { get { return _representation.count @@ -2153,7 +2139,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as trivially computable. public var regions: CollectionOfOne { return CollectionOfOne(self) } @@ -2168,7 +2154,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as a generic, trivially forwarding function. public func withUnsafeBytes(_ body: (UnsafeRawBufferPointer) throws -> ResultType) rethrows -> ResultType { return try _representation.withUnsafeBytes(body) } @@ -2184,7 +2170,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as a generic, trivially forwarding function. public mutating func withUnsafeMutableBytes(_ body: (UnsafeMutableRawBufferPointer) throws -> ResultType) rethrows -> ResultType { return try _representation.withUnsafeMutableBytes(body) } @@ -2197,14 +2183,14 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// - parameter pointer: A pointer to the buffer you wish to copy the bytes into. /// - parameter count: The number of bytes to copy. /// - warning: This method does not verify that the contents at pointer have enough space to hold `count` bytes. - @inlinable + @inlinable // This is @inlinable as trivially forwarding. public func copyBytes(to pointer: UnsafeMutablePointer, count: Int) { precondition(count >= 0, "count of bytes to copy must not be negative") if count == 0 { return } _copyBytesHelper(to: UnsafeMutableRawPointer(pointer), from: startIndex..<(startIndex + count)) } - @inlinable + @inlinable // This is @inlinable as trivially forwarding. internal func _copyBytesHelper(to pointer: UnsafeMutableRawPointer, from range: Range) { if range.isEmpty { return } _representation.copyBytes(to: pointer, from: range) @@ -2215,7 +2201,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// - parameter pointer: A pointer to the buffer you wish to copy the bytes into. /// - parameter range: The range in the `Data` to copy. /// - warning: This method does not verify that the contents at pointer have enough space to hold the required number of bytes. - @inlinable + @inlinable // This is @inlinable as trivially forwarding. public func copyBytes(to pointer: UnsafeMutablePointer, from range: Range) { _copyBytesHelper(to: pointer, from: range) } @@ -2227,7 +2213,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// - parameter buffer: A buffer to copy the data into. /// - parameter range: A range in the data to copy into the buffer. If the range is empty, this function will return 0 without copying anything. If the range is nil, as much data as will fit into `buffer` is copied. /// - returns: Number of bytes copied into the destination buffer. - @inlinable + @inlinable // This is @inlinable as generic and reasonably small. public func copyBytes(to buffer: UnsafeMutableBufferPointer, from range: Range? = nil) -> Int { let cnt = count guard cnt > 0 else { return 0 } @@ -2293,7 +2279,6 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// - parameter range: The range of this data in which to perform the search. Default value is `nil`, which means the entire content of this data. /// - returns: A `Range` specifying the location of the found data, or nil if a match could not be found. /// - precondition: `range` must be in the bounds of the Data. - @inlinable public func range(of dataToFind: Data, options: Data.SearchOptions = [], in range: Range? = nil) -> Range? { let nsRange : NSRange if let r = range { @@ -2319,19 +2304,18 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl _representation.enumerateBytes(block) } - @inlinable + @inlinable // This is @inlinable as a generic, trivially forwarding function. internal mutating func _append(_ buffer : UnsafeBufferPointer) { if buffer.isEmpty { return } _representation.append(contentsOf: UnsafeRawBufferPointer(buffer)) } - @inlinable + @inlinable // This is @inlinable as a generic, trivially forwarding function. public mutating func append(_ bytes: UnsafePointer, count: Int) { if count == 0 { return } _append(UnsafeBufferPointer(start: bytes, count: count)) } - @inlinable public mutating func append(_ other: Data) { guard other.count > 0 else { return } other.withUnsafeBytes { (buffer: UnsafeRawBufferPointer) in @@ -2342,19 +2326,19 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// Append a buffer of bytes to the data. /// /// - parameter buffer: The buffer of bytes to append. The size is calculated from `SourceType` and `buffer.count`. - @inlinable + @inlinable // This is @inlinable as a generic, trivially forwarding function. public mutating func append(_ buffer : UnsafeBufferPointer) { _append(buffer) } - @inlinable + @inlinable // This is @inlinable as trivially forwarding. public mutating func append(contentsOf bytes: [UInt8]) { bytes.withUnsafeBufferPointer { (buffer: UnsafeBufferPointer) -> Void in _append(buffer) } } - @inlinable + @inlinable // This is @inlinable as an important generic funnel point, despite being non-trivial. public mutating func append(contentsOf elements: S) where S.Element == Element { // If the sequence is already contiguous, access the underlying raw memory directly. if let contiguous = elements as? ContiguousBytes { @@ -2388,12 +2372,22 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl // Copy the contents of the buffer... _representation.append(contentsOf: UnsafeRawBufferPointer(start: base, count: endIndex)) - /// ... and append the rest byte-wise. + // ... and append the rest byte-wise, buffering through an InlineData. + var buffer = InlineData() while let element = iter.next() { - Swift.withUnsafeBytes(of: element) { - _representation.append(contentsOf: $0) + if buffer.count < buffer.capacity { + buffer.append(byte: element) + } else { + buffer.withUnsafeBytes { _representation.append(contentsOf: $0) } + buffer.count = 0 } } + + // If we've still got bytes left in the buffer (i.e. the loop ended before we filled up the buffer and cleared it out), append them. + if buffer.count > 0 { + buffer.withUnsafeBytes { _representation.append(contentsOf: $0) } + buffer.count = 0 + } } } @@ -2403,7 +2397,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// /// If `range` exceeds the bounds of the data, then the data is resized to fit. /// - parameter range: The range in the data to set to `0`. - @inlinable + @inlinable // This is @inlinable as trivially forwarding. public mutating func resetBytes(in range: Range) { // it is worth noting that the range here may be out of bounds of the Data itself (which triggers a growth) precondition(range.lowerBound >= 0, "Ranges must not be negative bounds") @@ -2418,7 +2412,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// - precondition: The bounds of `subrange` must be valid indices of the collection. /// - parameter subrange: The range in the data to replace. If `subrange.lowerBound == data.count && subrange.count == 0` then this operation is an append. /// - parameter data: The replacement data. - @inlinable + @inlinable // This is @inlinable as trivially forwarding. public mutating func replaceSubrange(_ subrange: Range, with data: Data) { data.withUnsafeBytes { (buffer: UnsafeRawBufferPointer) in _representation.replaceSubrange(subrange, with: buffer.baseAddress, count: buffer.count) @@ -2432,7 +2426,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// - precondition: The bounds of `subrange` must be valid indices of the collection. /// - parameter subrange: The range in the data to replace. /// - parameter buffer: The replacement bytes. - @inlinable + @inlinable // This is @inlinable as a generic, trivially forwarding function. public mutating func replaceSubrange(_ subrange: Range, with buffer: UnsafeBufferPointer) { guard !buffer.isEmpty else { return } replaceSubrange(subrange, with: buffer.baseAddress!, count: buffer.count * MemoryLayout.stride) @@ -2445,7 +2439,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// - precondition: The bounds of `subrange` must be valid indices of the collection. /// - parameter subrange: The range in the data to replace. /// - parameter newElements: The replacement bytes. - @inlinable + @inlinable // This is @inlinable as generic and reasonably small. public mutating func replaceSubrange(_ subrange: Range, with newElements: ByteCollection) where ByteCollection.Iterator.Element == Data.Iterator.Element { let totalCount = Int(newElements.count) _withStackOrHeapBuffer(totalCount) { conditionalBuffer in @@ -2459,7 +2453,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as trivially forwarding. public mutating func replaceSubrange(_ subrange: Range, with bytes: UnsafeRawPointer, count cnt: Int) { _representation.replaceSubrange(subrange, with: bytes, count: cnt) } @@ -2467,7 +2461,6 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// Return a new copy of the data in a specified range. /// /// - parameter range: The range to copy. - @inlinable public func subdata(in range: Range) -> Data { if isEmpty || range.upperBound - range.lowerBound == 0 { return Data() @@ -2486,7 +2479,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// /// - parameter options: The options to use for the encoding. Default value is `[]`. /// - returns: The Base-64 encoded string. - @inlinable + @inlinable // This is @inlinable as trivially forwarding. public func base64EncodedString(options: Data.Base64EncodingOptions = []) -> String { return _representation.withInteriorPointerReference { return $0.base64EncodedString(options: options) @@ -2497,7 +2490,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// /// - parameter options: The options to use for the encoding. Default value is `[]`. /// - returns: The Base-64 encoded data. - @inlinable + @inlinable // This is @inlinable as trivially forwarding. public func base64EncodedData(options: Data.Base64EncodingOptions = []) -> Data { return _representation.withInteriorPointerReference { return $0.base64EncodedData(options: options) @@ -2508,12 +2501,11 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl // /// The hash value for the data. - @inlinable - public var hashValue: Int { - return _representation.hashValue + @inline(never) // This is not inlinable as emission into clients could cause cross-module inconsistencies if they are not all recompiled together. + public func hash(into hasher: inout Hasher) { + _representation.hash(into: &hasher) } - @inlinable public func advanced(by amount: Int) -> Data { let length = count - amount precondition(length > 0) @@ -2528,7 +2520,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl // MARK: Index and Subscript /// Sets or returns the byte at the specified index. - @inlinable + @inlinable // This is @inlinable as trivially forwarding. public subscript(index: Index) -> UInt8 { get { return _representation[index] @@ -2538,7 +2530,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as trivially forwarding. public subscript(bounds: Range) -> Data { get { return _representation[bounds] @@ -2548,7 +2540,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable + @inlinable // This is @inlinable as a generic, trivially forwarding function. public subscript(_ rangeExpression: R) -> Data where R.Bound: FixedWidthInteger { get { @@ -2572,7 +2564,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } /// The start `Index` in the data. - @inlinable + @inlinable // This is @inlinable as trivially forwarding. public var startIndex: Index { get { return _representation.startIndex @@ -2582,31 +2574,31 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// The end `Index` into the data. /// /// This is the "one-past-the-end" position, and will always be equal to the `count`. - @inlinable + @inlinable // This is @inlinable as trivially forwarding. public var endIndex: Index { get { return _representation.endIndex } } - @inlinable + @inlinable // This is @inlinable as trivially computable. public func index(before i: Index) -> Index { return i - 1 } - @inlinable + @inlinable // This is @inlinable as trivially computable. public func index(after i: Index) -> Index { return i + 1 } - @inlinable + @inlinable // This is @inlinable as trivially computable. public var indices: Range { get { return startIndex..) -> (Iterator, UnsafeMutableBufferPointer.Index) { guard !isEmpty else { return (makeIterator(), buffer.startIndex) } let cnt = Swift.min(count, buffer.count) @@ -2621,7 +2613,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl /// An iterator over the contents of the data. /// /// The iterator will increment byte-by-byte. - @inlinable + @inlinable // This is @inlinable as trivially computable. public func makeIterator() -> Data.Iterator { return Iterator(self, at: startIndex) } @@ -2639,7 +2631,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl @usableFromInline internal var _idx: Data.Index @usableFromInline internal let _endIdx: Data.Index - @usableFromInline + @usableFromInline // This is @usableFromInline as a non-trivial initializer. internal init(_ data: Data, at loc: Data.Index) { // The let vars prevent this from being marked as @inlinable _data = data @@ -2655,7 +2647,6 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl } } - @inlinable public mutating func next() -> UInt8? { let idx = _idx let bufferSize = MemoryLayout.size @@ -2699,7 +2690,7 @@ public struct Data : ReferenceConvertible, Equatable, Hashable, RandomAccessColl public var mutableBytes: UnsafeMutableRawPointer { fatalError() } /// Returns `true` if the two `Data` arguments are equal. - @inlinable + @inlinable // This is @inlinable as emission into clients is safe -- the concept of equality on Data will not change. public static func ==(d1 : Data, d2 : Data) -> Bool { let length1 = d1.count if length1 != d2.count { diff --git a/test/stdlib/TestData.swift b/test/stdlib/TestData.swift index ec8606f6e1550..50de8f1bd8805 100644 --- a/test/stdlib/TestData.swift +++ b/test/stdlib/TestData.swift @@ -673,12 +673,6 @@ class TestData : TestDataSuper { expectEqual("SGVsbG8gV29ybGQ=", base64, "trivial base64 conversion should work") } - func test_dataHash() { - let dataStruct = "Hello World".data(using: .utf8)! - let dataObj = dataStruct as NSData - expectEqual(dataObj.hashValue, dataStruct.hashValue, "Data and NSData should have the same hash value") - } - func test_base64Data_medium() { let data = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut at tincidunt arcu. Suspendisse nec sodales erat, sit amet imperdiet ipsum. Etiam sed ornare felis. Nunc mauris turpis, bibendum non lectus quis, malesuada placerat turpis. Nam adipiscing non massa et semper. Nulla convallis semper bibendum. Aliquam dictum nulla cursus mi ultricies, at tincidunt mi sagittis. Nulla faucibus at dui quis sodales. Morbi rutrum, dui id ultrices venenatis, arcu urna egestas felis, vel suscipit mauris arcu quis risus. Nunc venenatis ligula at orci tristique, et mattis purus pulvinar. Etiam ultricies est odio. Nunc eleifend malesuada justo, nec euismod sem ultrices quis. Etiam nec nibh sit amet lorem faucibus dapibus quis nec leo. Praesent sit amet mauris vel lacus hendrerit porta mollis consectetur mi. Donec eget tortor dui. Morbi imperdiet, arcu sit amet elementum interdum, quam nisl tempor quam, vitae feugiat augue purus sed lacus. In ac urna adipiscing purus venenatis volutpat vel et metus. Nullam nec auctor quam. Phasellus porttitor felis ac nibh gravida suscipit tempus at ante. Nunc pellentesque iaculis sapien a mattis. Aenean eleifend dolor non nunc laoreet, non dictum massa aliquam. Aenean quis turpis augue. Praesent augue lectus, mollis nec elementum eu, dignissim at velit. Ut congue neque id ullamcorper pellentesque. Maecenas euismod in elit eu vehicula. Nullam tristique dui nulla, nec convallis metus suscipit eget. Cras semper augue nec cursus blandit. Nulla rhoncus et odio quis blandit. Praesent lobortis dignissim velit ut pulvinar. Duis interdum quam adipiscing dolor semper semper. Nunc bibendum convallis dui, eget mollis magna hendrerit et. Morbi facilisis, augue eu fringilla convallis, mauris est cursus dolor, eu posuere odio nunc quis orci. Ut eu justo sem. Phasellus ut erat rhoncus, faucibus arcu vitae, vulputate erat. Aliquam nec magna viverra, interdum est vitae, rhoncus sapien. Duis tincidunt tempor ipsum ut dapibus. Nullam commodo varius metus, sed sollicitudin eros. Etiam nec odio et dui tempor blandit posuere.".data(using: .utf8)! let base64 = data.base64EncodedString() @@ -1189,6 +1183,49 @@ class TestData : TestDataSuper { expectEqual(Data(bytes: [1]), slice) } + // This test uses `repeatElement` to produce a sequence -- the produced sequence reports its actual count as its `.underestimatedCount`. + func test_appendingNonContiguousSequence_exactCount() { + var d = Data() + + // d should go from .empty representation to .inline. + // Appending a small enough sequence to fit in .inline should actually be copied. + d.append(contentsOf: 0x00...0x01) + expectEqual(Data([0x00, 0x01]), d) + + // Appending another small sequence should similarly still work. + d.append(contentsOf: 0x02...0x02) + expectEqual(Data([0x00, 0x01, 0x02]), d) + + // If we append a sequence of elements larger than a single InlineData, the internal append here should buffer. + // We want to make sure that buffering in this way does not accidentally drop trailing elements on the floor. + d.append(contentsOf: 0x03...0x17) + expectEqual(Data([0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17]), d) + } + + // This test is like test_appendingNonContiguousSequence_exactCount but uses a sequence which reports 0 for its `.underestimatedCount`. + // This attempts to hit the worst-case scenario of `Data.append(_:)` -- a discontiguous sequence of unknown length. + func test_appendingNonContiguousSequence_underestimatedCount() { + var d = Data() + + // d should go from .empty representation to .inline. + // Appending a small enough sequence to fit in .inline should actually be copied. + d.append(contentsOf: (0x00...0x01).makeIterator()) // `.makeIterator()` produces a sequence whose `.underestimatedCount` is 0. + expectEqual(Data([0x00, 0x01]), d) + + // Appending another small sequence should similarly still work. + d.append(contentsOf: (0x02...0x02).makeIterator()) // `.makeIterator()` produces a sequence whose `.underestimatedCount` is 0. + expectEqual(Data([0x00, 0x01, 0x02]), d) + + // If we append a sequence of elements larger than a single InlineData, the internal append here should buffer. + // We want to make sure that buffering in this way does not accidentally drop trailing elements on the floor. + d.append(contentsOf: (0x03...0x17).makeIterator()) // `.makeIterator()` produces a sequence whose `.underestimatedCount` is 0. + expectEqual(Data([0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17]), d) + } + func test_sequenceInitializers() { let seq = repeatElement(UInt8(0x02), count: 3) // ensure we fall into the sequence case @@ -3728,7 +3765,6 @@ DataTests.test("testCopyBytes_oversized") { TestData().testCopyBytes_oversized() DataTests.test("testCopyBytes_ranges") { TestData().testCopyBytes_ranges() } DataTests.test("test_base64Data_small") { TestData().test_base64Data_small() } DataTests.test("test_base64Data_medium") { TestData().test_base64Data_medium() } -DataTests.test("test_dataHash") { TestData().test_dataHash() } DataTests.test("test_discontiguousEnumerateBytes") { TestData().test_discontiguousEnumerateBytes() } DataTests.test("test_basicReadWrite") { TestData().test_basicReadWrite() } DataTests.test("test_writeFailure") { TestData().test_writeFailure() } @@ -3760,6 +3796,8 @@ DataTests.test("test_copyBytes1") { TestData().test_copyBytes1() } DataTests.test("test_copyBytes2") { TestData().test_copyBytes2() } DataTests.test("test_sliceOfSliceViaRangeExpression") { TestData().test_sliceOfSliceViaRangeExpression() } DataTests.test("test_appendingSlices") { TestData().test_appendingSlices() } +DataTests.test("test_appendingNonContiguousSequence_exactCount") { TestData().test_appendingNonContiguousSequence_exactCount() } +DataTests.test("test_appendingNonContiguousSequence_underestimatedCount") { TestData().test_appendingNonContiguousSequence_underestimatedCount() } DataTests.test("test_sequenceInitializers") { TestData().test_sequenceInitializers() } DataTests.test("test_reversedDataInit") { TestData().test_reversedDataInit() } DataTests.test("test_replaceSubrangeReferencingMutable") { TestData().test_replaceSubrangeReferencingMutable() }