Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: got log data for some pipeline runs from the dashboard api
Browse files Browse the repository at this point in the history
Nathaniel-Xu committed Aug 27, 2024
1 parent 98b8855 commit a65bc0d
Showing 4 changed files with 68 additions and 1 deletion.
12 changes: 12 additions & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
@@ -519,6 +519,7 @@
"dagre": "^0.8.5",
"fs-extra": "^11.2.0",
"hbs": "^4.2.0",
"ky": "^1.7.1",
"langchain": "^0.2.16",
"marked": "^14.0.0",
"svg-pan-zoom": "github:bumbu/svg-pan-zoom",
38 changes: 37 additions & 1 deletion src/services/chatService.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import * as vscode from 'vscode';
import { TokenJS } from 'token.js';
import axios from 'axios';
import {
PipelineDataProvider,
PipelineRunTreeItem,
@@ -13,6 +14,10 @@ import { EnvironmentDataProvider } from '../views/activityBar/environmentView/En
import { LSClient } from './LSClient';
import { DagArtifact, DagStep, PipelineRunDag } from '../types/PipelineTypes';
import { JsonObject } from '../views/panel/panelView/PanelTreeItem';
import { ZenmlGlobalConfigResp } from 'type_hints'
import { ZenmlStoreConfig } from '../types/LSClientResponseTypes';
import { Z_SYNC_FLUSH } from 'zlib';
import { setPriority } from 'os';

export class ChatService {
private static instance: ChatService;
@@ -166,6 +171,25 @@ export class ChatService {
return `Stack Data:\n${contextString}\n`;
}

private async getPipelineRunNodes(type: string) {
let pipelineData = PipelineDataProvider.getInstance().getPipelineData();
let lsClient = LSClient.getInstance();
let dagData = await Promise.all(pipelineData.map(async (node: PipelineTreeItem) => {
let dag = await lsClient.sendLsClientRequest<PipelineRunDag>('getPipelineRunDag', [node.id]);
return dag
}));
let stepData = await Promise.all(dagData.map(async (dag: PipelineRunDag) => {
let filteredNodes = await Promise.all(dag.nodes.map(async (node: DagArtifact|DagStep) => {
if (type === "all" || node.type === type) {
return await lsClient.sendLsClientRequest<JsonObject>('getPipelineRunStep', [node.id]);
}
return null;
}));
return filteredNodes.filter((value) => value !== null);
}));
return stepData
}

private async getPanelData(): Promise<string> {
//Retrieve the run data through ls client requests
//TODO:
@@ -190,6 +214,18 @@ export class ChatService {

private async getLogData() {
let lsClient = LSClient.getInstance();
return JSON.stringify(await lsClient.sendLsClientRequest('getGlobalConfig'))
let dashboardUrl: string = ServerDataProvider.getInstance().getCurrentStatus().dashboard_url;
let apiToken: string = (await lsClient.sendLsClientRequest<ZenmlGlobalConfigResp>('getGlobalConfig')).store.api_token
let pipelineRunSteps = await this.getPipelineRunNodes('step')
let logs = await Promise.all(pipelineRunSteps[0].map(async (step) => {
let response = await axios.get(`${dashboardUrl}/api/v1/steps/${step.id}/logs`, {
headers: {
Authorization: `Bearer ${apiToken}`,
'accept': 'application/json'
}
})
return response.data
}))
return logs
}
}
18 changes: 18 additions & 0 deletions src/types/LSClientResponseTypes.ts
Original file line number Diff line number Diff line change
@@ -52,3 +52,21 @@ export interface ActiveStackResponse {

export type SetActiveStackResponse = ActiveStackResponse | ErrorMessageResponse;
export type GetActiveStackResponse = ActiveStackResponse | ErrorMessageResponse;

/***** Global Config Type *****/

export interface ZenmlGlobalConfigResp {
user_id: string,
user_email: string,
analytics_opt_in: boolean,
version: string,
active_stack_id: string,
active_workspace_name: string,
store: ZenmlStoreConfig
}

export interface ZenmlStoreConfig {
type: string,
url: string,
api_token?: string
}

0 comments on commit a65bc0d

Please sign in to comment.