Skip to content

Commit

Permalink
WIP,DNM: use axios-retry for agent and API comms
Browse files Browse the repository at this point in the history
  • Loading branch information
@adityamaru
adityamaru committed Dec 9, 2024
1 parent a415504 commit 1bad5d9
Show file tree
Hide file tree
Showing 3 changed files with 89 additions and 182 deletions.
200 changes: 85 additions & 115 deletions src/reporter.ts
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,50 @@
import * as core from '@actions/core';
import axios, {AxiosError, AxiosInstance, AxiosResponse} from 'axios';
import axiosRetry from 'axios-retry';
import {ExportRecordResponse} from '@docker/actions-toolkit/lib/types/buildx/history';
import * as utils from './utils';

// Configure base axios instance for Blacksmith API.
const createBlacksmithAPIClient = () => {
const apiUrl = process.env.BLACKSMITH_ENV?.includes('staging')
? 'https://stagingapi.blacksmith.sh'
: 'https://api.blacksmith.sh';

const client = axios.create({
baseURL: apiUrl,
headers: {
Authorization: `Bearer ${process.env.BLACKSMITH_STICKYDISK_TOKEN}`,
'X-Github-Repo-Name': process.env.GITHUB_REPO_NAME || '',
'Content-Type': 'application/json'
}
});

axiosRetry(client, {
retries: 5,
retryDelay: axiosRetry.exponentialDelay,
retryCondition: (error: AxiosError) => {
return axiosRetry.isNetworkOrIdempotentRequestError(error) ||
(error.response?.status ? error.response.status >= 500 : false);
}
});

return client;
};

export async function reportBuildPushActionFailure(error?: Error) {
const requestOptions = {
stickydisk_key: process.env.GITHUB_REPO_NAME || '',
repo_name: process.env.GITHUB_REPO_NAME || '',
region: process.env.BLACKSMITH_REGION || 'eu-central',
arch: process.env.BLACKSMITH_ENV?.includes('arm') ? 'arm64' : 'amd64',
vm_id: process.env.VM_ID || '',
petname: process.env.PETNAME || '',
message: error?.message || ''
};
const retryCondition = (error: AxiosError) => {
return error.response?.status ? error.response.status > 500 : false;
};
const response = await postWithRetryToBlacksmithAPI('/stickydisks/report-failed', requestOptions, retryCondition);
return response.data;
}
const requestOptions = {
stickydisk_key: process.env.GITHUB_REPO_NAME || '',
repo_name: process.env.GITHUB_REPO_NAME || '',
region: process.env.BLACKSMITH_REGION || 'eu-central',
arch: process.env.BLACKSMITH_ENV?.includes('arm') ? 'arm64' : 'amd64',
vm_id: process.env.VM_ID || '',
petname: process.env.PETNAME || '',
message: error?.message || ''
};

const client = createBlacksmithAPIClient();
const response = await client.post('/stickydisks/report-failed', requestOptions);
return response.data;
}

export async function reportBuildCompleted(exportRes?: ExportRecordResponse, blacksmithDockerBuildId?: string | null, buildRef?: string, dockerBuildDurationSeconds?: string, exposeId?: string): Promise<void> {
if (!blacksmithDockerBuildId) {
Expand All @@ -27,17 +53,18 @@ export async function reportBuildCompleted(exportRes?: ExportRecordResponse, bla
}

try {
const client = await utils.getBlacksmithAgentClient();
const agentClient = await getBlacksmithAgentClient();
const formData = new FormData();
formData.append('shouldCommit', 'true');
formData.append('vmID', process.env.VM_ID || '');
formData.append('exposeID', exposeId || '');
formData.append('stickyDiskKey', process.env.GITHUB_REPO_NAME || '');
const retryCondition = (error: AxiosError) => {
return error.response?.status ? error.response.status > 500 : false;
};

await postWithRetry(client, '/stickydisks', formData, retryCondition);
await agentClient.post('/stickydisks', formData, {
headers: {
'Content-Type': 'multipart/form-data'
}
});

// Report success to Blacksmith API
const requestOptions = {
Expand All @@ -48,13 +75,11 @@ export async function reportBuildCompleted(exportRes?: ExportRecordResponse, bla

if (exportRes) {
let buildRefSummary;
// Extract just the ref ID from the full buildRef path
const refId = buildRef?.split('/').pop();
core.info(`Using buildRef ID: ${refId}`);
if (refId && exportRes.summaries[refId]) {
buildRefSummary = exportRes.summaries[refId];
} else {
// Take first summary if buildRef not found
const summaryKeys = Object.keys(exportRes.summaries);
if (summaryKeys.length > 0) {
buildRefSummary = exportRes.summaries[summaryKeys[0]];
Expand All @@ -67,11 +92,10 @@ export async function reportBuildCompleted(exportRes?: ExportRecordResponse, bla
}
}

await postWithRetryToBlacksmithAPI(`/stickydisks/dockerbuilds/${blacksmithDockerBuildId}`, requestOptions, retryCondition);
return;
const client = createBlacksmithAPIClient();
await client.post(`/stickydisks/dockerbuilds/${blacksmithDockerBuildId}`, requestOptions);
} catch (error) {
core.warning('Error reporting build completed:', error);
// We don't want to fail the build if this fails so we swallow the error
}
}

Expand All @@ -82,17 +106,18 @@ export async function reportBuildFailed(dockerBuildId: string | null, dockerBuil
}

try {
const client = await utils.getBlacksmithAgentClient();
const blacksmithAgentClient = await getBlacksmithAgentClient();
const formData = new FormData();
formData.append('shouldCommit', 'false');
formData.append('vmID', process.env.VM_ID || '');
formData.append('exposeID', exposeId || '');
formData.append('stickyDiskKey', process.env.GITHUB_REPO_NAME || '');
const retryCondition = (error: AxiosError) => {
return error.response?.status ? error.response.status > 500 : false;
};

await postWithRetry(client, '/stickydisks', formData, retryCondition);
await blacksmithAgentClient.post('/stickydisks', formData, {
headers: {
'Content-Type': 'multipart/form-data'
}
});

// Report failure to Blacksmith API
const requestOptions = {
Expand All @@ -101,8 +126,8 @@ export async function reportBuildFailed(dockerBuildId: string | null, dockerBuil
runtime_seconds: dockerBuildDurationSeconds
};

await postWithRetryToBlacksmithAPI(`/stickydisks/dockerbuilds/${dockerBuildId}`, requestOptions, retryCondition);
return;
const blacksmithAPIClient = createBlacksmithAPIClient();
await blacksmithAPIClient.post(`/stickydisks/dockerbuilds/${dockerBuildId}`, requestOptions);
} catch (error) {
core.warning('Error reporting build failed:', error);
// We don't want to fail the build if this fails so we swallow the error
Expand All @@ -122,10 +147,8 @@ export async function reportBuild(dockerfilePath: string) {
git_branch: process.env.GITHUB_REF_NAME || ''
};
core.debug(`Reporting build with options: ${JSON.stringify(requestBody, null, 2)}`);
const retryCondition = (error: AxiosError) => {
return error.response?.status ? error.response.status > 500 : false;
};
const response = await postWithRetryToBlacksmithAPI('/stickydisks/dockerbuilds', requestBody, retryCondition);
const blacksmithAPIClient = createBlacksmithAPIClient();
const response = await blacksmithAPIClient.post('/stickydisks/dockerbuilds', requestBody);
return response.data;
} catch (error) {
const statusCode = (error as AxiosError)?.response?.status;
Expand All @@ -135,85 +158,32 @@ export async function reportBuild(dockerfilePath: string) {
}
}

async function postWithRetryToBlacksmithAPI(url: string, requestBody: unknown, retryCondition: (error: AxiosError) => boolean): Promise<AxiosResponse> {
const maxRetries = 5;
const retryDelay = 100;
const apiUrl = process.env.BLACKSMITH_ENV?.includes('staging') ? 'https://stagingapi.blacksmith.sh' : 'https://api.blacksmith.sh';

for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
core.debug(`Request headers: Authorization: Bearer ${process.env.BLACKSMITH_STICKYDISK_TOKEN}, X-Github-Repo-Name: ${process.env.GITHUB_REPO_NAME || ''}`);

const fullUrl = `${apiUrl}${url}`;
core.debug(`Making request to full URL: ${fullUrl}`);

return await axios.post(fullUrl, requestBody, {
headers: {
Authorization: `Bearer ${process.env.BLACKSMITH_STICKYDISK_TOKEN}`,
'X-Github-Repo-Name': process.env.GITHUB_REPO_NAME || '',
'Content-Type': 'application/json'
}
});
} catch (error) {
if (attempt === maxRetries || !retryCondition(error as AxiosError)) {
throw error;
}
core.warning(`Request failed, retrying (${attempt}/${maxRetries})...`);
await new Promise(resolve => setTimeout(resolve, retryDelay));
export async function getBlacksmithAgentClient(): Promise<AxiosInstance> {
const stickyDiskMgrUrl = 'http://192.168.127.1:5556';
const client = axios.create({
baseURL: stickyDiskMgrUrl
});

axiosRetry(client, {
retries: 5,
retryDelay: axiosRetry.exponentialDelay,
retryCondition: (error) => {
return axiosRetry.isNetworkOrIdempotentRequestError(error) ||
(error.response?.status ? error.response.status >= 500 : false);
}
}
throw new Error('Max retries reached');
}
});

async function postWithRetry(client: AxiosInstance, url: string, formData: FormData, retryCondition: (error: AxiosError) => boolean): Promise<AxiosResponse> {
const maxRetries = 5;
const retryDelay = 100;

for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
return await client.post(url, formData, {
headers: {
Authorization: `Bearer ${process.env.BLACKSMITH_STICKYDISK_TOKEN}`,
'X-Github-Repo-Name': process.env.GITHUB_REPO_NAME || '',
'Content-Type': 'multipart/form-data'
}
});
} catch (error) {
if (attempt === maxRetries || !retryCondition(error as AxiosError)) {
throw error;
}
core.warning(`Request failed, retrying (${attempt}/${maxRetries})...`);
await new Promise(resolve => setTimeout(resolve, retryDelay));
}
}
throw new Error('Max retries reached');
return client;
}

export async function getWithRetry(client: AxiosInstance, url: string, formData: FormData | null, retryCondition: (error: AxiosError) => boolean, options?: {signal?: AbortSignal}): Promise<AxiosResponse> {
const maxRetries = 5;
const retryDelay = 100;

for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
if (formData) {
return await client.get(url, {
data: formData,
headers: {
Authorization: `Bearer ${process.env.BLACKSMITH_STICKYDISK_TOKEN}`,
'X-Github-Repo-Name': process.env.GITHUB_REPO_NAME || '',
'Content-Type': 'multipart/form-data'
},
signal: options?.signal
});
}
return await client.get(url, {signal: options?.signal});
} catch (error) {
if (attempt === maxRetries || !retryCondition(error as AxiosError)) {
throw error;
}
core.warning(`Request failed, retrying (${attempt}/${maxRetries})...`);
await new Promise(resolve => setTimeout(resolve, retryDelay));
}
}
throw new Error('Max retries reached');
}
export async function get(client: AxiosInstance, url: string, formData: FormData | null, options?: {signal?: AbortSignal}): Promise<AxiosResponse> {
return await client.get(url, {
...(formData && {data: formData}),
headers: {
Authorization: `Bearer ${process.env.BLACKSMITH_STICKYDISK_TOKEN}`,
'X-Github-Repo-Name': process.env.GITHUB_REPO_NAME || '',
'Content-Type': 'multipart/form-data'
},
signal: options?.signal
});
}
12 changes: 4 additions & 8 deletions src/setup_builder.ts
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
import * as fs from 'fs';
import * as core from '@actions/core';
import {AxiosError} from 'axios';
import {exec} from 'child_process';
import {promisify} from 'util';
import * as TOML from '@iarna/toml';
import {Inputs} from './context';
import * as reporter from './reporter';
import * as utils from './utils';

const mountPoint = '/var/lib/buildkit';
const execAsync = promisify(exec);
Expand Down Expand Up @@ -146,8 +143,8 @@ async function getDiskSize(device: string): Promise<number> {
}
}

async function getStickyDisk(retryCondition: (error: AxiosError) => boolean, options?: {signal?: AbortSignal}): Promise<{expose_id: string; device: string}> {
const client = await utils.getBlacksmithAgentClient();
async function getStickyDisk(options?: {signal?: AbortSignal}): Promise<{expose_id: string; device: string}> {
const client = await reporter.getBlacksmithAgentClient();
const formData = new FormData();
// TODO(adityamaru): Support a stickydisk-per-build flag that will namespace the stickydisks by Dockerfile.
// For now, we'll use the repo name as the stickydisk key.
Expand All @@ -164,7 +161,7 @@ async function getStickyDisk(retryCondition: (error: AxiosError) => boolean, opt
for (const pair of formData.entries()) {
core.debug(`${pair[0]}: ${pair[1]}`);
}
const response = await reporter.getWithRetry(client, '/stickydisks', formData, retryCondition, options);
const response = await reporter.get(client, '/stickydisks', formData, options);
const exposeId = response.data?.expose_id || '';
const device = response.data?.disk_identifier || '';
return {expose_id: exposeId, device: device};
Expand Down Expand Up @@ -197,14 +194,13 @@ export async function startAndConfigureBuildkitd(parallelism: number, device: st
// throws an error if it is unable to do so because of a timeout or an error
export async function setupStickyDisk(dockerfilePath: string): Promise<{device: string; buildId?: string | null; exposeId: string}> {
try {
const retryCondition = (error: AxiosError) => (error.response?.status ? error.response.status >= 500 : error.code === 'ECONNRESET');
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 10000);

let buildResponse: {docker_build_id: string} | null = null;
let exposeId: string = '';
let device: string = '';
const stickyDiskResponse = await getStickyDisk(retryCondition, {signal: controller.signal});
const stickyDiskResponse = await getStickyDisk({signal: controller.signal});
exposeId = stickyDiskResponse.expose_id;
device = stickyDiskResponse.device;
if (device === '') {
Expand Down
59 changes: 0 additions & 59 deletions src/utils.ts
View file

This file was deleted.

0 comments on commit 1bad5d9

Please sign in to comment.