From 6a8aeb18be5bc4cc31cbcbdb91fe16a700f47051 Mon Sep 17 00:00:00 2001 From: secustor Date: Sat, 13 Jul 2024 00:43:22 +0200 Subject: [PATCH 1/5] chore(http/github): add utility function to fetch raw files --- lib/util/http/github.spec.ts | 20 ++++++++++++++++++++ lib/util/http/github.ts | 8 ++++++++ 2 files changed, 28 insertions(+) diff --git a/lib/util/http/github.spec.ts b/lib/util/http/github.spec.ts index 68af7bc831436d..6220b55921bf3c 100644 --- a/lib/util/http/github.spec.ts +++ b/lib/util/http/github.spec.ts @@ -786,4 +786,24 @@ describe('util/http/github', () => { ).rejects.toThrow(EXTERNAL_HOST_ERROR); }); }); + + describe('getRawFile()', () => { + it('add header and return', async () => { + httpMock + .scope(githubApiHost) + .get('/foo/bar/contents/lore/ipsum.txt') + .matchHeader( + 'accept', + 'application/vnd.github.raw+json, application/vnd.github.v3+json', + ) + .reply(200, 'foo'); + await expect( + githubApi.getRawFile( + `${githubApiHost}/foo/bar/contents/lore/ipsum.txt`, + ), + ).resolves.toMatchObject({ + body: 'foo', + }); + }); + }); }); diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts index 3db9d7b4a741dc..d250ed4e610b25 100644 --- a/lib/util/http/github.ts +++ b/lib/util/http/github.ts @@ -493,4 +493,12 @@ export class GithubHttp extends Http { return result; } + + public getRawFile(url: string): Promise { + return this.get(url, { + headers: { + accept: 'application/vnd.github.raw+json', + }, + }); + } } From 70cad8be6588e23d6df6fb829f650d0405782c17 Mon Sep 17 00:00:00 2001 From: secustor Date: Sat, 13 Jul 2024 10:03:04 +0200 Subject: [PATCH 2/5] test relative path --- lib/util/http/github.spec.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/util/http/github.spec.ts b/lib/util/http/github.spec.ts index 6220b55921bf3c..6dff8557c4d5ab 100644 --- a/lib/util/http/github.spec.ts +++ b/lib/util/http/github.spec.ts @@ -805,5 +805,19 @@ describe('util/http/github', () => { body: 'foo', }); }); + + it('support relative path', async () => { + httpMock + .scope(githubApiHost) + .get('/foo/bar/contents/lore/ipsum.txt') + .reply(200, 'foo'); + await expect( + githubApi.getRawFile( + `${githubApiHost}/foo/bar/contents/foo/../lore/ipsum.txt`, + ), + ).resolves.toMatchObject({ + body: 'foo', + }); + }); }); }); From df3887511a13f50afd7d7ed0d27d115ead6588e6 Mon Sep 17 00:00:00 2001 From: secustor Date: Mon, 15 Jul 2024 23:59:47 +0200 Subject: [PATCH 3/5] add test for baseUrl and path --- lib/util/http/github.spec.ts | 39 ++++++++++++++++++++++++++++++++++++ lib/util/http/github.ts | 6 +++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/lib/util/http/github.spec.ts b/lib/util/http/github.spec.ts index 6dff8557c4d5ab..2a7d5d7cea8e73 100644 --- a/lib/util/http/github.spec.ts +++ b/lib/util/http/github.spec.ts @@ -810,6 +810,10 @@ describe('util/http/github', () => { httpMock .scope(githubApiHost) .get('/foo/bar/contents/lore/ipsum.txt') + .matchHeader( + 'accept', + 'application/vnd.github.raw+json, application/vnd.github.v3+json', + ) .reply(200, 'foo'); await expect( githubApi.getRawFile( @@ -819,5 +823,40 @@ describe('util/http/github', () => { body: 'foo', }); }); + + it('support default to api.github.com if no baseURL has been supplied', async () => { + httpMock + .scope(githubApiHost) + .get('/foo/bar/contents/lore/ipsum.txt') + .matchHeader( + 'accept', + 'application/vnd.github.raw+json, application/vnd.github.v3+json', + ) + .reply(200, 'foo'); + await expect( + githubApi.getRawFile(`foo/bar/contents/lore/ipsum.txt`), + ).resolves.toMatchObject({ + body: 'foo', + }); + }); + + it('support custom host if a baseURL has been supplied', async () => { + const customApiHost = 'https://my.comapny.com/api/v3/'; + httpMock + .scope(customApiHost) + .get('/foo/bar/contents/lore/ipsum.txt') + .matchHeader( + 'accept', + 'application/vnd.github.raw+json, application/vnd.github.v3+json', + ) + .reply(200, 'foo'); + await expect( + githubApi.getRawFile(`foo/bar/contents/lore/ipsum.txt`, { + baseUrl: customApiHost, + }), + ).resolves.toMatchObject({ + body: 'foo', + }); + }); }); }); diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts index d250ed4e610b25..dc7091ec56020a 100644 --- a/lib/util/http/github.ts +++ b/lib/util/http/github.ts @@ -494,8 +494,12 @@ export class GithubHttp extends Http { return result; } - public getRawFile(url: string): Promise { + public getRawFile( + url: string, + options: InternalHttpOptions & GithubHttpOptions = {}, + ): Promise { return this.get(url, { + ...options, headers: { accept: 'application/vnd.github.raw+json', }, From 1902fccf74a6d68f35b7423e17e79e2d76468e52 Mon Sep 17 00:00:00 2001 From: secustor Date: Tue, 16 Jul 2024 00:17:54 +0200 Subject: [PATCH 4/5] support providing only file path with the repository --- lib/util/http/github.spec.ts | 38 ++++++++++++++++++++++++++++++++++++ lib/util/http/github.ts | 12 ++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/lib/util/http/github.spec.ts b/lib/util/http/github.spec.ts index 2a7d5d7cea8e73..5d29a552acf44a 100644 --- a/lib/util/http/github.spec.ts +++ b/lib/util/http/github.spec.ts @@ -858,5 +858,43 @@ describe('util/http/github', () => { body: 'foo', }); }); + + it('support default to api.github.com if no baseURL, but repository has been supplied', async () => { + httpMock + .scope(githubApiHost) + .get('/foo/bar/contents/lore/ipsum.txt') + .matchHeader( + 'accept', + 'application/vnd.github.raw+json, application/vnd.github.v3+json', + ) + .reply(200, 'foo'); + await expect( + githubApi.getRawFile(`lore/ipsum.txt`, { + repository: 'foo/bar', + }), + ).resolves.toMatchObject({ + body: 'foo', + }); + }); + + it('support custom host if a baseURL and repository has been supplied', async () => { + const customApiHost = 'https://my.comapny.com/api/v3/'; + httpMock + .scope(customApiHost) + .get('/foo/bar/contents/lore/ipsum.txt') + .matchHeader( + 'accept', + 'application/vnd.github.raw+json, application/vnd.github.v3+json', + ) + .reply(200, 'foo'); + await expect( + githubApi.getRawFile(`lore/ipsum.txt`, { + baseUrl: customApiHost, + repository: 'foo/bar', + }), + ).resolves.toMatchObject({ + body: 'foo', + }); + }); }); }); diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts index dc7091ec56020a..33aa90297a6d5c 100644 --- a/lib/util/http/github.ts +++ b/lib/util/http/github.ts @@ -498,11 +498,19 @@ export class GithubHttp extends Http { url: string, options: InternalHttpOptions & GithubHttpOptions = {}, ): Promise { - return this.get(url, { + const newOptions: InternalHttpOptions & GithubHttpOptions = { ...options, headers: { accept: 'application/vnd.github.raw+json', }, - }); + }; + + let newURL = url; + const httpRegex = regEx(/^https?:\/\//); + if (options.repository && !httpRegex.test(options.repository)) { + newURL = joinUrlParts(options.repository, 'contents', url); + } + + return this.get(newURL, newOptions); } } From 1b7336b75063e25367ae480e48c8fa23b53f1633 Mon Sep 17 00:00:00 2001 From: secustor Date: Thu, 1 Aug 2024 00:03:38 +0200 Subject: [PATCH 5/5] add suggestions --- lib/util/http/github.spec.ts | 64 ++++++++++++++++++++++++++++++++---- lib/util/http/github.ts | 21 ++++++++++-- 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/lib/util/http/github.spec.ts b/lib/util/http/github.spec.ts index 5d29a552acf44a..31292e7b3a0487 100644 --- a/lib/util/http/github.spec.ts +++ b/lib/util/http/github.spec.ts @@ -1,3 +1,4 @@ +import { Buffer } from 'node:buffer'; import { codeBlock } from 'common-tags'; import { DateTime } from 'luxon'; import * as httpMock from '../../../test/http-mock'; @@ -798,7 +799,7 @@ describe('util/http/github', () => { ) .reply(200, 'foo'); await expect( - githubApi.getRawFile( + githubApi.getRawTextFile( `${githubApiHost}/foo/bar/contents/lore/ipsum.txt`, ), ).resolves.toMatchObject({ @@ -816,7 +817,7 @@ describe('util/http/github', () => { ) .reply(200, 'foo'); await expect( - githubApi.getRawFile( + githubApi.getRawTextFile( `${githubApiHost}/foo/bar/contents/foo/../lore/ipsum.txt`, ), ).resolves.toMatchObject({ @@ -834,7 +835,7 @@ describe('util/http/github', () => { ) .reply(200, 'foo'); await expect( - githubApi.getRawFile(`foo/bar/contents/lore/ipsum.txt`), + githubApi.getRawTextFile(`foo/bar/contents/lore/ipsum.txt`), ).resolves.toMatchObject({ body: 'foo', }); @@ -851,7 +852,7 @@ describe('util/http/github', () => { ) .reply(200, 'foo'); await expect( - githubApi.getRawFile(`foo/bar/contents/lore/ipsum.txt`, { + githubApi.getRawTextFile(`foo/bar/contents/lore/ipsum.txt`, { baseUrl: customApiHost, }), ).resolves.toMatchObject({ @@ -869,7 +870,7 @@ describe('util/http/github', () => { ) .reply(200, 'foo'); await expect( - githubApi.getRawFile(`lore/ipsum.txt`, { + githubApi.getRawTextFile(`lore/ipsum.txt`, { repository: 'foo/bar', }), ).resolves.toMatchObject({ @@ -888,7 +889,7 @@ describe('util/http/github', () => { ) .reply(200, 'foo'); await expect( - githubApi.getRawFile(`lore/ipsum.txt`, { + githubApi.getRawTextFile(`lore/ipsum.txt`, { baseUrl: customApiHost, repository: 'foo/bar', }), @@ -896,5 +897,56 @@ describe('util/http/github', () => { body: 'foo', }); }); + + it('support default to api.github.com if content path is used', async () => { + httpMock + .scope(githubApiHost) + .get('/foo/bar/contents/lore/ipsum.txt') + .matchHeader( + 'accept', + 'application/vnd.github.raw+json, application/vnd.github.v3+json', + ) + .reply(200, 'foo'); + await expect( + githubApi.getRawTextFile(`foo/bar/contents/lore/ipsum.txt`), + ).resolves.toMatchObject({ + body: 'foo', + }); + }); + + it('support custom host if content path is used', async () => { + const customApiHost = 'https://my.comapny.com/api/v3/'; + httpMock + .scope(customApiHost) + .get('/foo/bar/contents/lore/ipsum.txt') + .matchHeader( + 'accept', + 'application/vnd.github.raw+json, application/vnd.github.v3+json', + ) + .reply(200, 'test'); + await expect( + githubApi.getRawTextFile(`foo/bar/contents/lore/ipsum.txt`, { + baseUrl: customApiHost, + }), + ).resolves.toMatchObject({ + body: 'test', + }); + }); + + it('throw error if a ', async () => { + httpMock + .scope(githubApiHost) + .get('/foo/bar/contents/lore/ipsum.bin') + .matchHeader( + 'accept', + 'application/vnd.github.raw+json, application/vnd.github.v3+json', + ) + .reply(200, Buffer.from('foo', 'binary')); + await expect( + githubApi.getRawTextFile(`foo/bar/contents/lore/ipsum.bin`, { + responseType: 'buffer', + }), + ).rejects.toThrow(); + }); }); }); diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts index 33aa90297a6d5c..56db54db38716c 100644 --- a/lib/util/http/github.ts +++ b/lib/util/http/github.ts @@ -494,7 +494,18 @@ export class GithubHttp extends Http { return result; } - public getRawFile( + /** + * Get the raw text file from a URL. + * Only use this method to fetch text files. + * + * @param url Full API URL, contents path or path inside the repository to the file + * @param options + * + * @example url = 'https://api.github.com/repos/renovatebot/renovate/contents/package.json' + * @example url = 'renovatebot/renovate/contents/package.json' + * @example url = 'package.json' & options.repository = 'renovatebot/renovate' + */ + public async getRawTextFile( url: string, options: InternalHttpOptions & GithubHttpOptions = {}, ): Promise { @@ -511,6 +522,12 @@ export class GithubHttp extends Http { newURL = joinUrlParts(options.repository, 'contents', url); } - return this.get(newURL, newOptions); + const result = await this.get(newURL, newOptions); + if (!is.string(result.body)) { + throw new Error( + `Expected raw text file but received ${typeof result.body}`, + ); + } + return result; } }