From 1eb111cd7ac6a9b71321157e31cf03a9944928eb Mon Sep 17 00:00:00 2001 From: Kostis Stefanou Date: Fri, 13 Sep 2024 19:16:35 +0300 Subject: [PATCH] Add support of Vision API. --- Package.swift | 19 ++++ Vision/Sources/API/VisionAPI.swift | 51 ++++++++++ .../Sources/Models/AnnotateImageRequest.swift | 56 +++++++++++ .../Models/VisionAnnotateTextResponse.swift | 94 +++++++++++++++++++ Vision/Sources/VisionClient.swift | 63 +++++++++++++ Vision/Sources/VisionConfig.swift | 40 ++++++++ Vision/Sources/VisionError.swift | 47 ++++++++++ Vision/Sources/VisionRequest.swift | 81 ++++++++++++++++ 8 files changed, 451 insertions(+) create mode 100644 Vision/Sources/API/VisionAPI.swift create mode 100644 Vision/Sources/Models/AnnotateImageRequest.swift create mode 100644 Vision/Sources/Models/VisionAnnotateTextResponse.swift create mode 100644 Vision/Sources/VisionClient.swift create mode 100644 Vision/Sources/VisionConfig.swift create mode 100644 Vision/Sources/VisionError.swift create mode 100644 Vision/Sources/VisionRequest.swift diff --git a/Package.swift b/Package.swift index 2b85514..ced4c93 100644 --- a/Package.swift +++ b/Package.swift @@ -18,6 +18,7 @@ let package = Package( "PubSub", "SecretManager", "Storage", + "Vision" ] ), .library( @@ -48,6 +49,10 @@ let package = Package( name: "GoogleCloudPubSub", targets: ["PubSub"] ), + .library( + name: "GoogleCloudVision", + targets: ["Vision"] + ), ], dependencies: [ .package(url: "https://github.com/swift-server/async-http-client.git", from: "1.18.0"), @@ -104,6 +109,13 @@ let package = Package( ], path: "PubSub/Sources/" ), + .target( + name: "Vision", + dependencies: [ + .target(name: "Core") + ], + path: "Vision/Sources" + ), .testTarget( name: "CoreTests", dependencies: [ @@ -140,5 +152,12 @@ let package = Package( ], path: "PubSub/Tests/" ), + .testTarget( + name: "VisionTests", + dependencies: [ + .target(name: "Vision") + ], + path: "Vision/Tests/" + ) ] ) diff --git a/Vision/Sources/API/VisionAPI.swift b/Vision/Sources/API/VisionAPI.swift new file mode 100644 index 0000000..cb11754 --- /dev/null +++ b/Vision/Sources/API/VisionAPI.swift @@ -0,0 +1,51 @@ +// +// VisionAPI.swift +// +// +// Created by Kostis Stefanou on 12/9/24. +// + +import AsyncHTTPClient +import NIO +import Core +import Foundation + +public protocol VisionAPI { + func annotateImages(_ imageRequests: [AnnotateImageRequest]) -> EventLoopFuture +} + +public extension VisionAPI { + func annotateImage(_ imageRequest: AnnotateImageRequest) -> EventLoopFuture { + annotateImages([imageRequest]) + } +} + +public final class GoogleCloudVisionAPI: VisionAPI { + + let endpoint: String + let request: GoogleCloudVisionRequest + let encoder = JSONEncoder() + + init(request: GoogleCloudVisionRequest, + endpoint: String) { + self.request = request + self.endpoint = endpoint + } + + private var annotateImagesPath: String { + "\(endpoint)/v1/images:annotate" + } + + public func annotateImages(_ imageRequests: [AnnotateImageRequest]) -> EventLoopFuture { + do { + let bodyDict = [ + "requests": [imageRequests] + ] + + let body = try HTTPClient.Body.data(encoder.encode(bodyDict)) + return request.send(method: .POST, path: annotateImagesPath, body: body) + } catch { + return request.eventLoop.makeFailedFuture(error) + } + } +} diff --git a/Vision/Sources/Models/AnnotateImageRequest.swift b/Vision/Sources/Models/AnnotateImageRequest.swift new file mode 100644 index 0000000..effa899 --- /dev/null +++ b/Vision/Sources/Models/AnnotateImageRequest.swift @@ -0,0 +1,56 @@ +// +// AnnotateImageRequest.swift +// +// +// Created by Kostis Stefanou on 12/9/24. +// + +import Foundation + +public struct AnnotateImageRequest: Encodable { + + public let image: Image + public let features: [Feature] + + public init(image: Image, features: [Feature]) { + self.image = image + self.features = features + } + + public init(imageUri: String) { + self.image = .init(source: .init(imageUri: imageUri)) + self.features = [.init(type: .textDetection)] + } +} + +// MARK: - Image + +public extension AnnotateImageRequest { + + struct Image: Encodable { + let source: ImageResource + } +} + +// MARK: - ImageResource + +extension AnnotateImageRequest.Image { + + struct ImageResource: Encodable { + let imageUri: String + } +} + +// MARK: - Feature + +public extension AnnotateImageRequest { + + struct Feature: Encodable { + + enum FeatureType: String, Encodable { + case textDetection = "TEXT_DETECTION" + } + + let type: FeatureType + } +} diff --git a/Vision/Sources/Models/VisionAnnotateTextResponse.swift b/Vision/Sources/Models/VisionAnnotateTextResponse.swift new file mode 100644 index 0000000..cbb5b9b --- /dev/null +++ b/Vision/Sources/Models/VisionAnnotateTextResponse.swift @@ -0,0 +1,94 @@ +// +// VisionTextToImageResponse.swift +// +// +// Created by Kostis Stefanou on 12/9/24. +// + +import Core +import Foundation + +public struct VisionAnnotateImageResponse: GoogleCloudModel { + public let responses: [VisionAnnotateImageResponseBody] +} + +public struct VisionAnnotateImageResponseBody: GoogleCloudModel { + public let textAnnotations: [TextAnnotation] + public let fullTextAnnotation: FullTextAnnotation +} + +public struct FullTextAnnotation: GoogleCloudModel { + public let pages: [Page] + public let text: String +} + +public extension FullTextAnnotation { + + struct Page: GoogleCloudModel { + public let property: WordProperty + public let width, height: Int + public let blocks: [Block] + } +} + +public extension FullTextAnnotation.Page { + + struct Block: GoogleCloudModel { + public let boundingBox: Bounding + public let paragraphs: [Paragraph] + public let blockType: String + } +} + +public extension FullTextAnnotation.Page.Block { + + struct Paragraph: GoogleCloudModel { + public let boundingBox: Bounding + public let words: [Word] + } +} + +// MARK : - Common Models + +public struct Word: GoogleCloudModel { + public let property: WordProperty? + public let boundingBox: Bounding + public let symbols: [Symbol] +} + +public struct WordProperty: GoogleCloudModel { + public let detectedLanguages: [DetectedLanguage] +} + +public struct Symbol: GoogleCloudModel { + public let boundingBox: Bounding + public let text: String + public let property: SymbolProperty? +} + +public struct SymbolProperty: GoogleCloudModel { + let detectedBreak: DetectedBreak +} + +public struct TextAnnotation: GoogleCloudModel { + public let locale: String? + public let description: String + public let boundingPoly: Bounding +} + +public struct Bounding: GoogleCloudModel { + public let vertices: [Vertex] +} + +public struct DetectedLanguage: GoogleCloudModel { + public let languageCode: String + public let confidence: Double +} + +public struct Vertex: GoogleCloudModel { + let x, y: Int +} + +public struct DetectedBreak: GoogleCloudModel { + public let type: String +} diff --git a/Vision/Sources/VisionClient.swift b/Vision/Sources/VisionClient.swift new file mode 100644 index 0000000..ffbdca5 --- /dev/null +++ b/Vision/Sources/VisionClient.swift @@ -0,0 +1,63 @@ +// +// VisionClient.swift +// +// +// Created by Kostis Stefanou on 12/9/24. +// + +import Core +import Foundation +import AsyncHTTPClient +import NIO + +public final class GoogleCloudVisionClient { + + public var vision: VisionAPI + var translationRequest: GoogleCloudVisionRequest + + /// Initialize a client for interacting with the Google Cloud Translation API + /// - Parameter credentials: The Credentials to use when authenticating with the APIs + /// - Parameter config: The configuration for the Translation API + /// - Parameter httpClient: An `HTTPClient` used for making API requests. + /// - Parameter eventLoop: The EventLoop used to perform the work on. + /// - Parameter base: The base URL to use for the Translation API + public init(credentials: GoogleCloudCredentialsConfiguration, + config: GoogleCloudVisionConfiguration, + httpClient: HTTPClient, + eventLoop: EventLoop, + base: String = "https://vision.googleapis.com") throws { + /// A token implementing `OAuthRefreshable`. Loaded from credentials specified by `GoogleCloudCredentialsConfiguration`. + let refreshableToken = OAuthCredentialLoader.getRefreshableToken(credentials: credentials, + withConfig: config, + andClient: httpClient, + eventLoop: eventLoop) + + /// Set the projectId to use for this client. In order of priority: + /// - Environment Variable (GOOGLE_PROJECT_ID) + /// - Environment Variable (PROJECT_ID) + /// - Service Account's projectID + /// - `GoogleCloudTranslationConfigurations` `project` property (optionally configured). + /// - `GoogleCloudCredentialsConfiguration's` `project` property (optionally configured). + + guard let projectId = ProcessInfo.processInfo.environment["GOOGLE_PROJECT_ID"] ?? + ProcessInfo.processInfo.environment["PROJECT_ID"] ?? + (refreshableToken as? OAuthServiceAccount)?.credentials.projectId ?? + config.project ?? credentials.project else { + throw GoogleCloudVisionError.projectIdMissing + } + + translationRequest = GoogleCloudVisionRequest(httpClient: httpClient, + eventLoop: eventLoop, + oauth: refreshableToken, + project: projectId) + + vision = GoogleCloudVisionAPI(request: translationRequest, endpoint: base) + } + + /// Hop to a new eventloop to execute requests on. + /// - Parameter eventLoop: The eventloop to execute requests on. + public func hopped(to eventLoop: EventLoop) -> GoogleCloudVisionClient { + translationRequest.eventLoop = eventLoop + return self + } +} diff --git a/Vision/Sources/VisionConfig.swift b/Vision/Sources/VisionConfig.swift new file mode 100644 index 0000000..345c274 --- /dev/null +++ b/Vision/Sources/VisionConfig.swift @@ -0,0 +1,40 @@ +// +// VisionConfig.swift +// +// +// Created by Kostis Stefanou on 12/9/24. +// + +import Core + +public struct GoogleCloudVisionConfiguration: GoogleCloudAPIConfiguration { + public var scope: [GoogleCloudAPIScope] + public let serviceAccount: String + public let project: String? + public let subscription: String? = nil + + public init(scope: [GoogleCloudTranslationScope], serviceAccount: String, project: String?) { + self.scope = scope + self.serviceAccount = serviceAccount + self.project = project + } + + /// Create a new `GoogleCloudTranslationConfiguration` with cloud platform scope and the default service account. + public static func `default`() -> GoogleCloudVisionConfiguration { + return GoogleCloudVisionConfiguration(scope: [.cloudPlatform], + serviceAccount: "default", + project: nil) + } +} + +public enum GoogleCloudTranslationScope: GoogleCloudAPIScope { + /// View and manage your data across Google Cloud Platform services + + case cloudPlatform + + public var value: String { + switch self { + case .cloudPlatform: return "https://www.googleapis.com/auth/cloud-platform" + } + } +} diff --git a/Vision/Sources/VisionError.swift b/Vision/Sources/VisionError.swift new file mode 100644 index 0000000..39e229a --- /dev/null +++ b/Vision/Sources/VisionError.swift @@ -0,0 +1,47 @@ +// +// VisionError.swift +// +// +// Created by Kostis Stefanou on 12/9/24. +// + +import Core +import Foundation + +public enum GoogleCloudVisionError: GoogleCloudError { + case projectIdMissing + case unknownError(String) + + var localizedDescription: String { + switch self { + case .projectIdMissing: + return "Missing project id for GoogleCloudStorage API. Did you forget to set your project id?" + case .unknownError(let reason): + return "An unknown error occured: \(reason)" + } + } +} + +/// [Reference](https://cloud.google.com/vision/docs/reference/rest/v1/Status) +public struct VisionAPIError: GoogleCloudError, GoogleCloudModel { + /// An HTTP status code value, without the textual description. + public var code: Int + /// Description of the error. Same as `errors.message`. + public var message: String + ///A list of messages that carry the error details. There is a common set of message types for APIs to use. + public var details: [String: String] +} + +public struct VisionError: Codable { + /// The scope of the error. Example values include: global, push and usageLimits. + public var domain: String? + /// Example values include invalid, invalidParameter, and required. + public var reason: String? + /// Description of the error. + /// Example values include Invalid argument, Login required, and Required parameter: project. + public var message: String? + /// The location or part of the request that caused the error. Use with location to pinpoint the error. For example, if you specify an invalid value for a parameter, the locationType will be parameter and the location will be the name of the parameter. + public var locationType: String? + /// The specific item within the locationType that caused the error. For example, if you specify an invalid value for a parameter, the location will be the name of the parameter. + public var location: String? +} diff --git a/Vision/Sources/VisionRequest.swift b/Vision/Sources/VisionRequest.swift new file mode 100644 index 0000000..ea37e88 --- /dev/null +++ b/Vision/Sources/VisionRequest.swift @@ -0,0 +1,81 @@ +// +// VisionRequest.swift +// +// +// Created by Kostis Stefanou on 12/9/24. +// + +import Core +import Foundation +import NIO +import NIOFoundationCompat +import NIOHTTP1 +import AsyncHTTPClient + +class GoogleCloudVisionRequest: GoogleCloudAPIRequest { + + let refreshableToken: OAuthRefreshable + let project: String + let httpClient: HTTPClient + let responseDecoder: JSONDecoder = JSONDecoder() + var currentToken: OAuthAccessToken? + var tokenCreatedTime: Date? + var eventLoop: EventLoop + + init(httpClient: HTTPClient, eventLoop: EventLoop, oauth: OAuthRefreshable, project: String) { + self.refreshableToken = oauth + self.httpClient = httpClient + self.project = project + self.eventLoop = eventLoop + let dateFormatter = DateFormatter() + + dateFormatter.dateFormat = "yyyy-MM-dd'T'HH:mm:ss.SSSZ" + self.responseDecoder.dateDecodingStrategy = .formatted(dateFormatter) + } + + public func send(method: HTTPMethod, headers: HTTPHeaders = [:], path: String, query: String = "", body: HTTPClient.Body = .data(Data())) -> EventLoopFuture { + return withToken { token in + return self._send(method: method, headers: headers, path: path, query: query, body: body, accessToken: token.accessToken).flatMap { response in + do { + let model = try self.responseDecoder.decode(GCM.self, from: response) + return self.eventLoop.makeSucceededFuture(model) + } catch { + return self.eventLoop.makeFailedFuture(error) + } + } + } + } + + private func _send(method: HTTPMethod, headers: HTTPHeaders, path: String, query: String, body: HTTPClient.Body, accessToken: String) -> EventLoopFuture { + var _headers: HTTPHeaders = ["Authorization": "Bearer \(accessToken)", + "Content-Type": "application/json"] + headers.forEach { _headers.replaceOrAdd(name: $0.name, value: $0.value) } + + do { + let request = try HTTPClient.Request(url: "\(path)?\(query)", method: method, headers: _headers, body: body) + + return httpClient.execute(request: request, eventLoop: .delegate(on: self.eventLoop)).flatMap { response in + + guard var byteBuffer = response.body else { + fatalError("Response body from Google is missing! This should never happen.") + } + let responseData = byteBuffer.readData(length: byteBuffer.readableBytes)! + + guard (200...299).contains(response.status.code) else { + let error: Error + if let jsonError = try? self.responseDecoder.decode(VisionAPIError.self, from: responseData) { + error = jsonError + } else { + let body = response.body?.getString(at: response.body?.readerIndex ?? 0, length: response.body?.readableBytes ?? 0) ?? "" + error = VisionAPIError(code: Int(response.status.code), message: body, details: [:]) + } + + return self.eventLoop.makeFailedFuture(error) + } + return self.eventLoop.makeSucceededFuture(responseData) + } + } catch { + return self.eventLoop.makeFailedFuture(error) + } + } +}