Skip to content

Commit

Permalink
Simple HTTP Proxy (#49)
Browse files Browse the repository at this point in the history
* draft proxy

* fix proxy

* Decompress response before recording & remove header on forwarding to requester

Signed-off-by: Alexander Ignition <[email protected]>

* Update AppConfiguration

* Update ProxyMiddleware

* Update Loggers.swift

Add MultiplexLogHandler

* Add text proxy

* Update README.md

* Update Proxy example in README.md

Co-authored-by: Anton Glezman <[email protected]>

Co-authored-by: Vasily Fedorov <[email protected]>
Co-authored-by: Anton Glezman <[email protected]>
3 people authored Feb 15, 2022

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 787b830 commit b590424
Showing 14 changed files with 333 additions and 75 deletions.
26 changes: 14 additions & 12 deletions Packages/CatbirdApp/Sources/CatbirdApp/AppConfiguration.swift
Original file line number Diff line number Diff line change
@@ -1,20 +1,17 @@
import Foundation
import Vapor

/// Application configuration.
public struct AppConfiguration {

/// Application work mode.
public enum Mode: Equatable {
case write(URL)
case read
}
public let isRecordMode: Bool

/// Application work mode.
public let mode: Mode
public let proxyEnabled: Bool

/// The directory for mocks.
public let mocksDirectory: URL

public let redirectUrl: URL?

public let maxBodySize: String
}

@@ -38,11 +35,16 @@ extension AppConfiguration {
return url
}()

let isRecordMode = environment["CATBIRD_RECORD_MODE"].flatMap { NSString(string: $0).boolValue } ?? false
let proxyEnabled = environment["CATBIRD_PROXY_ENABLED"].flatMap { NSString(string: $0).boolValue } ?? false
let redirectUrl = environment["CATBIRD_REDIRECT_URL"].flatMap { URL(string: $0) }
let maxBodySize = environment["CATBIRD_MAX_BODY_SIZE", default: "50mb"]

if let path = environment["CATBIRD_PROXY_URL"], let url = URL(string: path) {
return AppConfiguration(mode: .write(url), mocksDirectory: mocksDirectory, maxBodySize: maxBodySize)
}
return AppConfiguration(mode: .read, mocksDirectory: mocksDirectory, maxBodySize: maxBodySize)
return AppConfiguration(
isRecordMode: isRecordMode,
proxyEnabled: proxyEnabled,
mocksDirectory: mocksDirectory,
redirectUrl: redirectUrl,
maxBodySize: maxBodySize)
}
}
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ struct FileDirectoryPath {
}

func preferredFileURL(for request: Request) -> URL {
var fileUrl = url.appendingPathComponent(request.url.string)
var fileUrl = fileURL(for: request)

guard fileUrl.pathExtension.isEmpty else {
return fileUrl
@@ -21,7 +21,7 @@ struct FileDirectoryPath {
}

func filePaths(for request: Request) -> [String] {
let fileUrl = url.appendingPathComponent(request.url.string)
let fileUrl = fileURL(for: request)

var urls: [URL] = []
if fileUrl.pathExtension.isEmpty {
@@ -32,4 +32,16 @@ struct FileDirectoryPath {
urls.append(fileUrl)
return urls.map { $0.absoluteString }
}

private func fileURL(for request: Request) -> URL {
var fileUrl = url
if let host = request.url.host {
fileUrl.appendPathComponent(host)
}
fileUrl.appendPathComponent(request.url.path)
if fileUrl.absoluteString.hasSuffix("/") {
fileUrl.appendPathComponent("index")
}
return fileUrl
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import Vapor

extension Request {
/// Send HTTP request.
///
/// - Parameter configure: client request configuration function.
/// - Returns: Server response.
func send(configure: ((inout ClientRequest) -> Void)? = nil) -> EventLoopFuture<Response> {
return body
.collect(max: nil)
.flatMap { (bytesBuffer: ByteBuffer?) -> EventLoopFuture<Response> in
var clientRequest = self.clientRequest(body: bytesBuffer)
configure?(&clientRequest)
return self.client.send(clientRequest).map { (clientResponse: ClientResponse) -> Response in
clientResponse.response(version: self.version)
}
}
}

/// Convert to HTTP client request.
private func clientRequest(body: ByteBuffer?) -> ClientRequest {
var headers = self.headers
if let host = headers.first(name: "Host") {
headers.replaceOrAdd(name: "X-Forwarded-Host", value: host)
headers.remove(name: "Host")
}
return ClientRequest(method: method, url: url, headers: headers, body: body)
}
}

extension HTTPHeaders {
fileprivate var contentLength: Int? {
first(name: "Content-Length").flatMap { Int($0) }
}
}

extension ClientResponse {
/// Convert to Server Response.
fileprivate func response(version: HTTPVersion) -> Response {
let body = body.map { Response.Body(buffer: $0) } ?? .empty
return Response(status: status, version: version, headers: headers, body: body)
}
}
5 changes: 4 additions & 1 deletion Packages/CatbirdApp/Sources/CatbirdApp/Common/Loggers.swift
Original file line number Diff line number Diff line change
@@ -13,7 +13,10 @@ enum Loggers {
return Logging.Logger(label: CatbirdInfo.current.domain)
#else
return Logging.Logger(label: CatbirdInfo.current.domain) {
OSLogHandler(subsystem: $0, category: category)
Logging.MultiplexLogHandler([
OSLogHandler(subsystem: $0, category: category),
Logging.StreamLogHandler.standardOutput(label: $0)
])
}
#endif
}
Original file line number Diff line number Diff line change
@@ -2,11 +2,11 @@ import Vapor

final class AnyMiddleware: Middleware {

private typealias Handler = (Request, Responder) -> EventLoopFuture<Response>
typealias Handler = (Request, Responder) -> EventLoopFuture<Response>

private let handler: Handler

private init(handler: @escaping Handler) {
init(handler: @escaping Handler) {
self.handler = handler
}

@@ -24,19 +24,9 @@ extension AnyMiddleware {
/// - Returns: A new `Middleware`.
static func notFound(_ handler: @escaping (Request) -> EventLoopFuture<Response>) -> Middleware {
AnyMiddleware { (request, responder) -> EventLoopFuture<Response> in
responder.respond(to: request)
.flatMap { (response: Response) -> EventLoopFuture<Response> in
if response.status == .notFound {
return handler(request)
}
return request.eventLoop.makeSucceededFuture(response)
}
.flatMapError { (error: Error) -> EventLoopFuture<Response> in
if let abort = error as? AbortError, abort.status == .notFound {
return handler(request)
}
return request.eventLoop.makeFailedFuture(error)
}
responder.respond(to: request).notFound {
handler(request)
}
}
}

@@ -49,3 +39,23 @@ extension AnyMiddleware {
}

}

extension EventLoopFuture where Value: Response {
func notFound(
_ handler: @escaping () -> EventLoopFuture<Response>
) -> EventLoopFuture<Response> {

return flatMap { [eventLoop] (response: Response) -> EventLoopFuture<Response> in
if response.status == .notFound {
return handler()
}
return eventLoop.makeSucceededFuture(response)
}
.flatMapError { [eventLoop] (error: Error) -> EventLoopFuture<Response> in
if let abort = error as? AbortError, abort.status == .notFound {
return handler()
}
return eventLoop.makeFailedFuture(error)
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import Vapor

final class ProxyMiddleware: Middleware {

func respond(to request: Request, chainingTo next: Responder) -> EventLoopFuture<Response> {
if request.url.host == nil {
request.logger.info("Proxy break \(request.method) \(request.url)")
return next.respond(to: request)
}
return next.respond(to: request).notFound {
var url = request.url
if url.scheme == nil {
url.scheme = url.port == 443 ? "https" : "http"
}

request.logger.info("Proxy \(request.method) \(url), scheme \(url.scheme ?? "<nil>")")

// Send request to real host
return request.send {
$0.url = url
}
}
}
}
Original file line number Diff line number Diff line change
@@ -11,29 +11,17 @@ final class RedirectMiddleware: Middleware {
// MARK: - Middleware

func respond(to request: Request, chainingTo next: Responder) -> EventLoopFuture<Response> {
return request.body.collect(max: nil).flatMap { (body: ByteBuffer?) -> EventLoopFuture<Response> in
var headers = request.headers
headers.remove(name: "Host")

var clientRequest = ClientRequest(
method: request.method,
url: self.redirectURI,
headers: headers,
body: request.body.data)
// Handle only direct requests to catbird
if request.url.host != nil {
return next.respond(to: request) // proxy request
}

clientRequest.url.string += request.url.string
var uri = redirectURI
uri.string += request.url.string

return request
.client
.send(clientRequest)
.map { (response: ClientResponse) -> Response in
let body = response.body.map { Response.Body(buffer: $0) } ?? .empty
return Response(
status: response.status,
version: request.version,
headers: response.headers,
body: body)
}
// Send request to redirect host
return request.send {
$0.url = uri
}
}
}
35 changes: 24 additions & 11 deletions Packages/CatbirdApp/Sources/CatbirdApp/configure.swift
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import CatbirdAPI
import Vapor
import NIOSSL

public struct CatbirdInfo: Content {
public static let current = CatbirdInfo(
@@ -28,26 +29,38 @@ public func configure(_ app: Application, _ configuration: AppConfiguration) thr
store: InMemoryResponseStore(),
logger: Loggers.inMemoryStore)

// MARK: - Register Middlewares
// MARK: - Register Middleware

// Pubic resource for web page
app.middleware.use(FileMiddleware(publicDirectory: app.directory.publicDirectory))
switch configuration.mode {
case .read:
app.logger.info("Read mode")
// try read from static mocks if route not found
app.middleware.use(AnyMiddleware.notFound(fileStore.response))
// try read from dynamic mocks
app.middleware.use(AnyMiddleware.notFound(inMemoryStore.response))
case .write(let url):
app.logger.info("Write mode")
if configuration.isRecordMode {
app.logger.info("Record mode")
app.http.client.configuration.decompression = .enabled(limit: .none)
// capture response and write to file
app.middleware.use(AnyMiddleware.capture { request, response in
if response.headers.contains(name: "Content-encoding") {
response.headers.remove(name: "Content-encoding")
}
let pattern = RequestPattern(method: .init(request.method.rawValue), url: request.url.string)
let mock = ResponseMock(status: Int(response.status.code), body: response.body.data)
return fileStore.perform(.update(pattern, mock), for: request).map { _ in response }
})
// redirect request to another server
// catch 404 and try read from real server
if configuration.proxyEnabled {
app.middleware.use(ProxyMiddleware())
}
} else {
app.logger.info("Read mode")
// catch 404 and try read from real server
if configuration.proxyEnabled {
app.middleware.use(ProxyMiddleware())
}
// try read from static mocks if route not found
app.middleware.use(AnyMiddleware.notFound(fileStore.response))
// try read from dynamic mocks
app.middleware.use(AnyMiddleware.notFound(inMemoryStore.response))
}
if let url = configuration.redirectUrl {
app.middleware.use(RedirectMiddleware(serverURL: url))
}

Original file line number Diff line number Diff line change
@@ -5,18 +5,24 @@ final class AppConfigurationTests: XCTestCase {

func testDetectReadMode() throws {
let config = try AppConfiguration.detect(from: [:])
XCTAssertEqual(config.mode, .read)
XCTAssertEqual(config.isRecordMode, false)
XCTAssertEqual(config.proxyEnabled, false)
XCTAssertEqual(config.mocksDirectory.absoluteString, AppConfiguration.sourceDir)
XCTAssertNil(config.redirectUrl)
XCTAssertEqual(config.maxBodySize, "50mb")
}

func testDetectWriteMode() throws {
let config = try AppConfiguration.detect(from: [
"CATBIRD_PROXY_URL": "/",
"CATBIRD_RECORD_MODE": "1",
"CATBIRD_PROXY_ENABLED": "1",
"CATBIRD_REDIRECT_URL": "https://example.com",
"CATBIRD_MAX_BODY_SIZE": "1kb"
])
XCTAssertEqual(config.mode, .write(URL(string: "/")!))
XCTAssertEqual(config.isRecordMode, true)
XCTAssertEqual(config.proxyEnabled, true)
XCTAssertEqual(config.mocksDirectory.absoluteString, AppConfiguration.sourceDir)
XCTAssertEqual(config.redirectUrl?.absoluteString, "https://example.com")
XCTAssertEqual(config.maxBodySize, "1kb")
}

12 changes: 9 additions & 3 deletions Packages/CatbirdApp/Tests/CatbirdAppTests/App/AppTestCase.swift
Original file line number Diff line number Diff line change
@@ -9,18 +9,24 @@ class AppTestCase: XCTestCase {
willSet { app?.shutdown() }
}

func setUpApp(mode: AppConfiguration.Mode) throws {
func setUpApp(
isRecordMode: Bool = false,
proxyEnabled: Bool = false,
redirectUrl: URL? = nil
) throws {
let config = AppConfiguration(
mode: mode,
isRecordMode: isRecordMode,
proxyEnabled: proxyEnabled,
mocksDirectory: URL(string: mocksDirectory)!,
redirectUrl: redirectUrl,
maxBodySize: "50kb")
app = Application(.testing)
try configure(app, config)
}

override func setUp() {
super.setUp()
XCTAssertNoThrow(try setUpApp(mode: .read))
XCTAssertNoThrow(try setUpApp())
XCTAssertEqual(app.routes.defaultMaxBodySize, 51200)
}

68 changes: 66 additions & 2 deletions Packages/CatbirdApp/Tests/CatbirdAppTests/App/AppTests.swift
Original file line number Diff line number Diff line change
@@ -16,8 +16,8 @@ final class AppTests: AppTestCase {
func testWriteFileMock() throws {
// Given
let api = JokeAPI()
XCTAssertNoThrow(try setUpApp(mode: .write(api.host)), """
Launch the app in redirect mode to \(api.host) and write files to a folder \(mocksDirectory)
XCTAssertNoThrow(try setUpApp(isRecordMode: true, redirectUrl: api.url), """
Launch the app in redirect mode to \(api.url) and write files to a folder \(mocksDirectory)
""")
addTeardownBlock {
let path = self.mocksDirectory + api.root
@@ -45,6 +45,38 @@ final class AppTests: AppTestCase {
}
}

func testWriteFileMockWithProxy() throws {
// Given
let api = JokeAPI()
let directory = "\(mocksDirectory)/\(api.host)"
XCTAssertNoThrow(try setUpApp(isRecordMode: true, proxyEnabled: true))
addTeardownBlock {
XCTAssertNotNil(try? FileManager.default.removeItem(atPath: directory), """
Remove created files and directories at \(directory)
""")
}

// When
for joke in api.jokes {
try app.test(.GET, joke.path, headers: api.headers, beforeRequest: { request in
request.url = URI(scheme: .http, host: api.host, path: request.url.path)
}, afterResponse: { response in
XCTAssertEqual(response.status.code, 200)
XCTAssertEqual(response.body.string, joke.text, """
Returned the joke by index \(joke.id)
""")
})
}

// Then
for joke in api.jokes {
let path = directory + joke.path + ".txt"
XCTAssertEqual(try String(contentsOfFile: path), joke.text, """
The joke by \(joke.id) was saved to a file at path \(path)
""")
}
}

func testAddMock() throws {
// Given
let mock = ResponseMock(status: 300, headers: ["X": "Y"], body: Data("hello".utf8))
@@ -87,6 +119,37 @@ final class AppTests: AppTestCase {
}
}

func testAddMockWithProxy() throws {
// Given
XCTAssertNoThrow(try setUpApp(isRecordMode: false, proxyEnabled: true))

let api = JokeAPI()
let mockJoke = api.jokes[0]
let proxyJoke = api.jokes[1]

let mock = ResponseMock(status: 200, body: Data(mockJoke.text.utf8))
let mockUrl = URI(scheme: .http, host: api.host, path: mockJoke.path).string
let pattern = RequestPattern(method: .GET, url: mockUrl)

// When
try app.perform(.update(pattern, mock))

// Then
try app.test(.GET, mockJoke.path, headers: api.headers, beforeRequest: { request in
request.url = URI(scheme: .http, host: api.host, path: request.url.path)
}, afterResponse: { response in
XCTAssertEqual(response.status.code, 200)
XCTAssertEqual(response.body.string, mockJoke.text, "mock response")
})

try app.test(.GET, proxyJoke.path, headers: api.headers, beforeRequest: { request in
request.url = URI(scheme: .http, host: api.host, path: request.url.path)
}, afterResponse: { response in
XCTAssertEqual(response.status.code, 200)
XCTAssertEqual(response.body.string, proxyJoke.text, "proxy response")
})
}

func testUpdateMock() throws {
// Given
var mock = ResponseMock(status: 200, body: Data("first".utf8))
@@ -294,3 +357,4 @@ final class AppTests: AppTestCase {
}
}
}

Original file line number Diff line number Diff line change
@@ -41,7 +41,7 @@ final class FileDirectoryPathTests: RequestTestCase {
func testFilePathsForRequestWithEmptyAccept() {
let path = FileDirectoryPath(url: URL(string: "fixtures")!)
let request = makeRequest(
url: "stores",
url: "/stores",
headers: ["Accept": "text/plain, application/json"]
)
XCTAssertEqual(path.filePaths(for: request), [
@@ -50,4 +50,48 @@ final class FileDirectoryPathTests: RequestTestCase {
"fixtures/stores"
])
}

func testRequestWithHost() {
let path = FileDirectoryPath(url: URL(string: "root")!)
let request = makeRequest(
url: "http://example.com/news.html"
)
XCTAssertEqual(
path.preferredFileURL(for: request),
URL(string: "root/example.com/news.html")!
)
XCTAssertEqual(path.filePaths(for: request), [
"root/example.com/news.html",
])
}

func testRequestWithSlash() {
let path = FileDirectoryPath(url: URL(string: "root")!)
let request = makeRequest(
url: "http://example.com/",
headers: ["Accept": "text/html"]
)
XCTAssertEqual(
path.preferredFileURL(for: request),
URL(string: "root/example.com/index.html")!
)
XCTAssertEqual(path.filePaths(for: request), [
"root/example.com/index.html",
"root/example.com/index",
])
}

func testRequestWithQuery() {
let path = FileDirectoryPath(url: URL(string: "root")!)
let request = makeRequest(
url: "http://example.com/item?data=123"
)
XCTAssertEqual(
path.preferredFileURL(for: request),
URL(string: "root/example.com/item")!
)
XCTAssertEqual(path.filePaths(for: request), [
"root/example.com/item",
])
}
}
12 changes: 10 additions & 2 deletions Packages/CatbirdApp/Tests/CatbirdAppTests/Helpers/JokeAPI.swift
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
import CatbirdApp
import Vapor
import Foundation

/*
curl http://icanhazdadjoke.com/j/R7UfaahVfFd
curl http://icanhazdadjoke.com/j/R7UfaahVfFd --proxy http://127.0.0.1:8080
*/

/// https://icanhazdadjoke.com/api
struct JokeAPI {
@@ -9,8 +15,11 @@ struct JokeAPI {
var path: String { "/j/\(id)" }
}

/// API url.
let url = URL(string: "https://icanhazdadjoke.com")!

/// API host.
let host = URL(string: "https://icanhazdadjoke.com")!
var host: String { url.host! }

/// API root directory.
let root = "/j"
@@ -31,5 +40,4 @@ struct JokeAPI {
id: "0ozAXv4Mmjb",
text: "Why did the tomato blush? Because it saw the salad dressing.")
]

}
39 changes: 37 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
@@ -208,9 +208,44 @@ $ xed .

## Environment variables

`CATBIRD_MOCKS_DIR` — Directory where static mocks are located.
- `CATBIRD_MOCKS_DIR` — Directory where static mocks are located.
- `CATBIRD_RECORD_MODE` — set this variable to `1` so that the application starts recording HTTP responses along the path set in `CATBIRD_MOCKS_DIR`. Default `0`.
- `CATBIRD_REDIRECT_URL` — set this url to forward direct requests to catbird. By default, nil. If the recording mode is not enabled, then first the responses will be searched in the mocks and only if nothing is found, the request will be forwarded.
- `CATBIRD_PROXY_ENABLED` — set this variable to `1` to forward proxy requests to catbird. By default, `0`. If the recording mode is not enabled, then first the responses will be searched in the mocks and only if nothing is found, the request will be proxied.

`CATBIRD_PROXY_URL` — If you specify this URL Catbird will run in write mode. In this mode, requests to Catbird will be redirected to the `CATBIRD_PROXY_URL`. Upon receipt of response from the server it will be written to the `CATBIRD_MOCKS_DIR` directory.
> Catbird supports proxying only HTTP requests. HTTPS requests are not supported!
### Redirect example

Run catbird with `CATBIRD_REDIRECT_URL`.

```bash
CATBIRD_REDIRECT_URL=https://api.github.com ./catbird
```

All direct requests will be forwarded to `CATBIRD_REDIRECT_URL`.

```bash
curl http://127.0.0.1:8080/zen
```

The response will be returned as to the request https://api.github.com/zen

### Proxy example

Run catbird with `CATBIRD_PROXY_ENABLED=1`.

```bash
CATBIRD_PROXY_ENABLED=1 ./catbird
```

By enabling this mode, the catbird will be running as a local http proxy server.
You can configure your http client to use this proxy, and all requests will be proxied thought the catbird and redirected to the real host.
It might be helpful if you don't want to change the base url of your requests.

```bash
curl http://api.github.com/zen --proxy http://127.0.0.1:8080
```

## Logs

0 comments on commit b590424

Please sign in to comment.