Develop (#15)

* fix: .gitignore add docker data fix: 修正發生部分錯誤時，資料應該還是可以正常回傳 fix: 調整 Logger & 其他 Bug ci: add cdk8s fix: ci fix: ci fix: cdk8s fix: ci fix: ci fix: ci fix: ci fix: ci fix: ci fix: ci fix: env fix: ci fix: ci * fix: cdk8s * update(openai): update openai to v3.3.0 fix: changelog
sd0xdev · Aug 17, 2023 · 972f487 · 972f487
1 parent 034f7cd
commit 972f487
Show file tree

Hide file tree

Showing 82 changed files with 75,699 additions and 1,058 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -32,3 +32,15 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ./coverage/apps/yggdrasil-core-engine/lcov.info,./coverage/apps/yggdrasil-discord-client/lcov.info,./coverage/libs/nest-openai-client/lcov.info,./coverage/libs/nest-winston/lcov.info,./coverage/libs/utils/lcov.info
+  push_to_gitlab:
+    # if branch is develop, push to gitlab develop branch
+    if: github.ref == 'refs/heads/develop'
+    runs-on: ubuntu-latest
+    needs: main
+    steps:
+      - name: Push to GitLab
+        uses: actions/checkout@v3
+      - name: Push to GitLab
+        run: |
+          git fetch --unshallow
+          git push https://yasuoyuhao:${{secrets.GITLAB_TOKEN}}@gitlab.com/welfree-studio/asgard-hub.git develop -f
diff --git a/.gitignore b/.gitignore
@@ -42,3 +42,6 @@ Thumbs.db
 
 # local env files
 .env.local
+
+#docker file
+docker-data/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -0,0 +1,144 @@
+stages:
+  - build-image
+  - cloud-deploy
+
+variables:
+  LANG: "en_US.UTF-8"
+  LC_ALL: "en_US.UTF-8"
+  GIT_DEPTH: 0
+  GIT_SUBMODULE_STRATEGY: normal
+  # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled
+  DOCKER_HOST: tcp://docker:2376
+  DOCKER_TLS_CERTDIR: "/certs"
+  # These are usually specified by the entrypoint, however the
+  # Kubernetes executor doesn't run entrypoints
+  # https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4125
+  DOCKER_TLS_VERIFY: 1
+  DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
+  DOCKER_DRIVER: overlay2
+
+build-and-push: &build-and-push
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
+  tags:
+    - self-docker
+  stage: build-image
+  needs: []
+  script:
+    - mkdir -p /kaniko/.docker
+    - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
+    - chmod +x ./tools/shell-script/package.sh
+    - sh ./tools/shell-script/package.sh
+  rules:
+    - when: never
+  retry:
+    max: 2
+    when:
+      - stuck_or_timeout_failure
+      - runner_system_failure
+
+build-and-push:staging:
+  <<: *build-and-push
+  stage: build-image
+  needs: []
+  variables:
+    PROJECT_NAME: yggdrasil-core-engine
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "develop"'
+  retry:
+    max: 2
+    when:
+      - stuck_or_timeout_failure
+      - runner_system_failure
+
+build-and-push-discord-client:staging:
+  <<: *build-and-push
+  stage: build-image
+  needs: []
+  variables:
+    PROJECT_NAME: yggdrasil-discord-client
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "develop"'
+  retry:
+    max: 2
+    when:
+      - stuck_or_timeout_failure
+      - runner_system_failure
+
+cloud-deploy: &cloud-deploy
+  image: alpine:3.14
+  stage: cloud-deploy
+  tags:
+    - self-docker
+  before_script:
+    - apk update  && apk add --no-cache curl nodejs npm
+    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
+    - chmod +x ./kubectl && mv ./kubectl /usr/local/bin/kubectl
+    - mkdir -p $HOME/.kube
+    - echo -n $KUBE_CONFIG | base64 -d > $HOME/.kube/config
+    - kubectl config view
+  script:
+    - echo $NODE_ENV
+    - echo $KUBE_NAME_SPACE
+    - echo $PROJECT_NAME
+    - kubectl delete jobs --all -n $KUBE_NAME_SPACE
+    - kubectl delete cronjobs --all -n $KUBE_NAME_SPACE
+    - chmod +x ./tools/shell-script/build-cdk8s.sh
+    - sh ./tools/shell-script/build-cdk8s.sh
+    - kubectl create secret docker-registry gitlab-registry --docker-server="$CI_REGISTRY" --docker-username="$CI_DEPLOY_USER" --docker-password="$CI_DEPLOY_PASSWORD" --docker-email="$GITLAB_USER_EMAIL" -n $KUBE_NAME_SPACE -o yaml --dry-run=client | kubectl apply -f -
+    - kubectl apply -f ./tools/cdk8s/${PROJECT_NAME}/dist/*.yaml -n $KUBE_NAME_SPACE
+  rules:
+    - when: never
+  artifacts:
+    name: "${PROJECT_NAME}-$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    paths:
+      - ./tools/cdk8s/${PROJECT_NAME}/dist/*
+    expire_in: 3 days
+  retry:
+    max: 2
+    when:
+      - stuck_or_timeout_failure
+      - runner_system_failure
+
+cloud-deploy:staging:
+  <<: *cloud-deploy
+  stage: cloud-deploy
+  needs: [
+    'build-and-push:staging'
+  ]
+  variables:
+    NODE_ENV: staging
+    KUBE_NAME_SPACE: yggdrasil-core-engine-staging
+    PROJECT_NAME: yggdrasil-core-engine
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "develop"'
+  environment:
+    name: staging
+    action: prepare
+  retry:
+    max: 2
+    when:
+      - stuck_or_timeout_failure
+      - runner_system_failure
+
+cloud-deploy-discord-client:staging:
+  <<: *cloud-deploy
+  stage: cloud-deploy
+  needs: [
+    'build-and-push-discord-client:staging'
+  ]
+  variables:
+    NODE_ENV: staging
+    KUBE_NAME_SPACE: yggdrasil-discord-client-staging
+    PROJECT_NAME: yggdrasil-discord-client
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "develop"'
+  environment:
+    name: staging
+    action: prepare
+  retry:
+    max: 2
+    when:
+      - stuck_or_timeout_failure
+      - runner_system_failure
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.0.11] - 2023/08/17
+
+### Added
+
+- `.github/workflows/main.yml`: Added a new job `push_to_gitlab` that pushes the code to the GitLab repository's develop branch if the current branch is `develop`. This job runs on Ubuntu and uses the `actions/checkout` action to fetch the latest code and then pushes it to GitLab using the provided GitLab token.
+
+- `.gitignore`: Added `docker-data/` to the `.gitignore` file to exclude the `docker-data` directory from version control.
+
+- `.gitlab-ci.yml`: Added a new GitLab CI configuration file with multiple stages and jobs for building and deploying the code. These jobs use various tags, scripts, and variables to determine the build and deployment steps. The stages include `build-image` and `cloud-deploy`, and the jobs include `build-and-push` and `cloud-deploy` with different variables and rules.
+
+### Fixed
+
+- `apps/yggdrasil-core-engine/.env.local.example`: Updated the environment variables in the example `.env.local` file. The variables `OPENAI_API_KEY`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_DEPLOYMENT_NAME`, and `AZURE_OPENAI_MODEL_NAME` were replaced with `CORE_ENGINE_API_KEY`, `CHATGPT_API_KEY`, `CHATGPT_ORG`, `OPENAI_API_TYPE`, `AZURE_API_BASE_PATH`, `AZURE_CHATGPT_API_KEY`, `AZURE_DEPLOYMENT_NAME`, `AZURE_API_VERSION`, `RPC_API_KEY`, and `MONGO_DB_URI`.
+
+- `apps/yggdrasil-core-engine/Dockerfile`: Added a Dockerfile for the `yggdrasil-core-engine` application. This Dockerfile includes multiple stages (`development`, `build`, and `production`) and sets up the necessary environment for building and running the application.
+
 ## [0.0.10] - 2023/06/16
 
 ### Added

diff --git a/apps/yggdrasil-core-engine/.env.local.example b/apps/yggdrasil-core-engine/.env.local.example
@@ -1,7 +1,12 @@
 PACKAGE_NAME=Yggdrasil Core Engine
 SERVICE_NAME=yggdrasil-core-engine
-OPENAI_API_KEY=
-AZURE_OPENAI_API_KEY=
-AZURE_OPENAI_ENDPOINT=
-AZURE_OPENAI_DEPLOYMENT_NAME=
-AZURE_OPENAI_MODEL_NAME=
+CORE_ENGINE_API_KEY=
+CHATGPT_API_KEY=
+CHATGPT_ORG=
+OPENAI_API_TYPE=
+AZURE_API_BASE_PATH=
+AZURE_CHATGPT_API_KEY=
+AZURE_DEPLOYMENT_NAME=
+AZURE_API_VERSION='2023-03-15-preview'
+RPC_API_KEY=
+MONGO_DB_URI=
diff --git a/apps/yggdrasil-core-engine/Dockerfile b/apps/yggdrasil-core-engine/Dockerfile
@@ -0,0 +1,34 @@
+FROM docker.io/node:18.16  AS development
+
+WORKDIR /usr/src/app
+
+COPY --chown=node:node package*.json ./
+
+RUN npm ci
+
+COPY --chown=node:node . .
+
+USER node
+
+FROM node:lts-slim As build
+
+WORKDIR /usr/src/app
+
+COPY --chown=node:node package*.json ./
+COPY --chown=node:node --from=development /usr/src/app/node_modules ./node_modules
+COPY --chown=node:node . .
+
+RUN npx nx run yggdrasil-core-engine:build
+RUN npm cache clean --force
+
+USER node
+
+FROM docker.io/node:18.16-alpine AS production
+
+COPY --chown=node:node --from=build /usr/src/app/dist/apps/yggdrasil-core-engine ./yggdrasil-core-engine
+COPY --chown=node:node --from=development /usr/src/app/node_modules ./node_modules
+
+ARG SERVER_VERSION
+ENV SERVER_VERSION=${SERVER_VERSION}
+
+CMD [ "node", "yggdrasil-core-engine" ]
diff --git a/apps/yggdrasil-core-engine/src/app/__mocks__/logger-helper.service.spec.ts b/apps/yggdrasil-core-engine/src/app/__mocks__/logger-helper.service.spec.ts
@@ -3,17 +3,6 @@ import { LoggerService } from '@nestjs/common';
 export function mockClearAll() {
   // Reset mockedLogger
   Object.values(innerMockedLogger).forEach((m) => m.mockClear());
-
-  // Reset LoggerHelperService
-  mockedLoggerHelperServiceMakeCreator.mockClear();
-  MockLoggerHelperService.mockClear();
-
-  // Reset TrackerLoggerCreator
-  mockedTrackerLoggerCreatorCreate.mockClear();
-  TrackerLoggerCreator.mockClear();
-
-  // Reset TrackerLogger
-  TrackerLogger.mockClear();
 }
 
 // Default mocked Logger
@@ -25,76 +14,13 @@ const innerMockedLogger = {
   verbose: jest.fn(),
 };
 
-export const mockedLogger: LoggerService = innerMockedLogger;
-
-// class LoggerHelperService
-export const mockedLoggerHelperServiceMakeCreator = jest.fn(function (
-  name: string
-) {
-  return new TrackerLoggerCreator(mockedLogger, name);
-});
-
-export const MockLoggerHelperService = jest.fn().mockImplementation(() => {
-  return {
-    create: mockedLoggerHelperServiceMakeCreator,
-  };
-});
-
-// class TrackerLoggerCreator
-export const mockedTrackerLoggerCreatorCreate = jest.fn(function (
-  trackingId: string
-) {
-  return new TrackerLogger(this.loggerService, this.name, trackingId);
-});
-
-export const TrackerLoggerCreator = jest
-  .fn()
-  .mockImplementation((logger: LoggerService, name: string) => {
-    return {
-      name,
-      loggerService: logger,
-      create: mockedTrackerLoggerCreatorCreate,
-    };
-  });
-
-// class TrackerLogger
-export const TrackerLogger = jest
-  .fn()
-  .mockImplementation(
-    (loggerService: LoggerService, name: string, _trackingId: string) => {
-      function createMessage(message: any) {
-        return {
-          trackingId: _trackingId,
-          message: `${name}: ${message}`,
-        };
-      }
-
-      const mocked = {
-        name,
-        loggerService,
-        _trackingId,
-        createMessage,
-        log: (message: any, context?: string) =>
-          loggerService.log(createMessage(message), context),
-        error: (message: any, context?: string) =>
-          loggerService.error(createMessage(message), context),
-        warn: (message: any, context?: string) =>
-          loggerService.warn(createMessage(message), context),
-        debug: (message: any, context?: string) =>
-          loggerService.debug(createMessage(message), context),
-        verbose: (message: any, context?: string) =>
-          loggerService.verbose(createMessage(message), context),
-      };
-
-      Object.defineProperty(mocked, 'trackingId', {
-        get: function () {
-          return this._trackingId;
-        },
-      });
-
-      return mocked;
-    }
-  );
+export class mockedLogger implements LoggerService {
+  log = innerMockedLogger.log;
+  error = innerMockedLogger.error;
+  warn = innerMockedLogger.warn;
+  debug = innerMockedLogger.debug;
+  verbose = innerMockedLogger.verbose;
+}
 
 it('should be defined', () => {
   expect(mockedLogger).toBeDefined();

diff --git a/apps/yggdrasil-core-engine/src/app/auth/auth.guard.ts b/apps/yggdrasil-core-engine/src/app/auth/auth.guard.ts
@@ -1,29 +1,21 @@
 import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common';
 import { Observable } from 'rxjs';
-import { AppService } from '../app.service';
-import {
-  TrackerLoggerCreator,
-  LoggerHelperService,
-} from '@asgard-hub/nest-winston';
+import { AsgardLogger } from '@asgard-hub/nest-winston';
 import { ConfigService } from '@nestjs/config';
 import { ConfigPath, IAppConfig } from '../config/app.config';
 
 @Injectable()
 export class AuthGuard implements CanActivate {
-  private readonly trackerLoggerCreator: TrackerLoggerCreator;
   constructor(
-    loggerHelperService: LoggerHelperService,
+    private readonly asgardLogger: AsgardLogger,
     private readonly configService: ConfigService
-  ) {
-    this.trackerLoggerCreator = loggerHelperService.create(AppService.name);
-  }
+  ) {}
   canActivate(
     context: ExecutionContext
   ): boolean | Promise<boolean> | Observable<boolean> {
-    const trackerLogger = this.trackerLoggerCreator.create('auth-guard');
     const metadata = context.getArgByIndex(1); // metadata
     if (!metadata) {
-      trackerLogger.warn('metadata is undefined');
+      this.asgardLogger.warn('metadata is undefined');
       return false;
     }
     const apiKey = metadata.get('authorization')[0];