Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support MinIO for file storage #365

Merged
merged 21 commits into from
Nov 26, 2024
Merged
Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
1acd5d9
feat: integrate MinIO for file storage
minai621 Oct 4, 2024
538e87a
feat: Add storage module for file upload
minai621 Oct 12, 2024
cdf4425
refactor: storage module to remove unnecessary credentials case by aw…
minai621 Oct 12, 2024
7c77472
refactor: storage module update condition check logic to use file_upl…
minai621 Oct 14, 2024
422284b
feat: set default configuration values for Minio
minai621 Oct 16, 2024
01418ae
refactor: update storage module to remove default value for AWS_REGION
minai621 Oct 16, 2024
926c3d7
refactor: remove unused StorageType enum
minai621 Oct 16, 2024
7992933
refactor: update .env.development to set default values for Minio
minai621 Oct 16, 2024
5c14681
feat: enhance file upload configuration for MinIO support
minai621 Nov 5, 2024
b8b9ae0
refactor: update .env.development with detailed comments for MinIO co…
minai621 Nov 11, 2024
c748dd8
refactor: update .env.development for MinIO configuration with specif…
minai621 Nov 11, 2024
23d6102
refactor: update docker-compose configuration for MinIO file upload
minai621 Nov 11, 2024
d3955fd
refactor: update file upload configuration type check in storage module
minai621 Nov 11, 2024
dfebd87
refactor: update file upload configuration to enable based on false c…
minai621 Nov 11, 2024
81244e4
feat: integrate MinIO for file storage
minai621 Oct 4, 2024
424170d
Feat configure MinIO for file storage and update environment variables
minai621 Nov 26, 2024
e55349e
Reformat code
devleejb Nov 26, 2024
d625868
Remove redundant `links` field in `docker-compose`
devleejb Nov 26, 2024
b15117f
Add dependant condition
devleejb Nov 26, 2024
867c4fe
Remove unused functions
devleejb Nov 26, 2024
057f843
Remove unused file
devleejb Nov 26, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 23 additions & 8 deletions backend/.env.development
Original file line number Diff line number Diff line change
Expand Up @@ -79,11 +79,26 @@ LANGCHAIN_API_KEY=your_langsmith_api_key_here
# To create a LangSmith project, visit LangSmith: https://www.langchain.com/langsmith
LANGCHAIN_PROJECT=your_langsmith_project_name_here

devleejb marked this conversation as resolved.
Show resolved Hide resolved

# FILE_UPLOAD: Whether to enable file upload to storage
# Set to true if file upload is required.
# If set to false, AWS_S3_BUCKET_NAME is not required.
FILE_UPLOAD=false
# AWS_S3_BUCKET_NAME: S3 Bucket name
# This is the name of the S3 Bucket
AWS_S3_BUCKET_NAME="your_s3_bucket_name"
# FILE_UPLOAD: Whether to enable file upload to storage.
# Available options: false, s3, minio.
# Set to "false" if file upload is not required.
# Set to "s3" or "minio" to enable file uploads.
FILE_UPLOAD=minio
devleejb marked this conversation as resolved.
Show resolved Hide resolved
# BUCKET_NAME: The name of the S3 or MinIO bucket to use.
# Required only if FILE_UPLOAD is set to "s3" or "minio".
BUCKET_NAME="default-storage"
# MINIO_ENDPOINT: The endpoint URL for the MinIO server.
# Format: http(s)://<host>:<port>.
# Example: http://localhost:9000 (For development mode).
MINIO_ENDPOINT="http://localhost:9000"
# MINIO_ACCESS_KEY: Access key for authentication.
# Default: minioadmin (for development only).
# Warning: Use a strong, unique value in production.
MINIO_ACCESS_KEY="minioadmin"
# MINIO_SECRET_KEY: Secret key for authentication.
# Default: minioadmin (for development only).
# Warning: Keep this value secret and never commit to version control.
MINIO_SECRET_KEY="minioadmin"
devleejb marked this conversation as resolved.
Show resolved Hide resolved
# AWS Region: only required for AWS S3
# Example: us-east-1
AWS_REGION="your_aws_region_here"
devleejb marked this conversation as resolved.
Show resolved Hide resolved
37 changes: 35 additions & 2 deletions backend/docker/docker-compose-full.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,22 @@ services:
LANGCHAIN_TRACING_V2: "false"
LANGCHAIN_API_KEY: "your_langsmith_api_key_here"
LANGCHAIN_PROJECT: "your_langsmith_project_name_here"
FILE_UPLOAD: false
AWS_S3_BUCKET_NAME: "your_s3_bucket_name"
FILE_UPLOAD: minio
AWS_REGION: "your_aws_region_here"
BUCKET_NAME: "default-storage"
MINIO_ENDPOINT: "http://localhost:9000"
minai621 marked this conversation as resolved.
Show resolved Hide resolved
MINIO_ACCESS_KEY: "minioadmin"
MINIO_SECRET_KEY: "minioadmin"
devleejb marked this conversation as resolved.
Show resolved Hide resolved
ports:
- "3000:3000"
depends_on:
- mongo
- minio
devleejb marked this conversation as resolved.
Show resolved Hide resolved
restart: unless-stopped
links:
- "mongo:mongo"
- "yorkie:yorkie"
- "minio:minio"
devleejb marked this conversation as resolved.
Show resolved Hide resolved

yorkie:
image: "yorkieteam/yorkie:0.5.6"
Expand Down Expand Up @@ -70,3 +76,30 @@ services:
interval: 5s
timeout: 2s
retries: 20

# You can remove the following content if you're using S3 or not using Minio.
minio:
image: minio/minio
ports:
- "9000:9000"
- "9001:9001"
environment:
MINIO_ROOT_USER: "minioadmin"
MINIO_ROOT_PASSWORD: "minioadmin"
command: server --console-address ":9001" --address ":9000" /data
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
devleejb marked this conversation as resolved.
Show resolved Hide resolved
interval: 30s
timeout: 20s
retries: 3
devleejb marked this conversation as resolved.
Show resolved Hide resolved

init_minio:
image: minio/mc
depends_on:
- minio
entrypoint: >
/bin/sh -c "
mc alias set codepair http://minio:9000 minioadmin minioadmin --api S3v4;
mc mb codepair/default-storage;
exit 0;
"
devleejb marked this conversation as resolved.
Show resolved Hide resolved
26 changes: 26 additions & 0 deletions backend/docker/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,29 @@ services:
interval: 5s
timeout: 2s
retries: 20

minio:
image: minio/minio
ports:
- "9000:9000"
- "9001:9001"
environment:
MINIO_ROOT_USER: "minioadmin"
MINIO_ROOT_PASSWORD: "minioadmin"
command: server --console-address ":9001" --address ":9000" /data
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 30s
timeout: 20s
retries: 3
devleejb marked this conversation as resolved.
Show resolved Hide resolved

init_minio:
image: minio/mc
depends_on:
- minio
entrypoint: >
/bin/sh -c "
mc alias set codepair http://minio:9000 minioadmin minioadmin --api S3v4;
mc mb codepair/default-storage;
exit 0;
"
devleejb marked this conversation as resolved.
Show resolved Hide resolved
18 changes: 10 additions & 8 deletions backend/src/app.module.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,20 @@
import { Module } from "@nestjs/common";
import { PrismaService } from "./db/prisma.service";
import { UsersModule } from "./users/users.module";
import { AuthModule } from "./auth/auth.module";
import { ConfigModule } from "@nestjs/config";
import { APP_GUARD } from "@nestjs/core/constants";
import { AuthModule } from "./auth/auth.module";
import { JwtAuthGuard } from "./auth/jwt.guard";
import { WorkspacesModule } from "./workspaces/workspaces.module";
import { WorkspaceUsersModule } from "./workspace-users/workspace-users.module";
import { WorkspaceDocumentsModule } from "./workspace-documents/workspace-documents.module";
import { DocumentsModule } from "./documents/documents.module";
import { CheckModule } from "./check/check.module";
import { PrismaService } from "./db/prisma.service";
import { DocumentsModule } from "./documents/documents.module";
import { FilesModule } from "./files/files.module";
import { IntelligenceModule } from "./intelligence/intelligence.module";
import { LangchainModule } from "./langchain/langchain.module";
import { FilesModule } from "./files/files.module";
import { SettingsModule } from "./settings/settings.module";
import { StorageModule } from "./storage/storage.module";
import { UsersModule } from "./users/users.module";
import { WorkspaceDocumentsModule } from "./workspace-documents/workspace-documents.module";
import { WorkspaceUsersModule } from "./workspace-users/workspace-users.module";
import { WorkspacesModule } from "./workspaces/workspaces.module";

@Module({
imports: [
Expand All @@ -34,6 +35,7 @@ import { SettingsModule } from "./settings/settings.module";
FilesModule,
ConfigModule,
SettingsModule,
StorageModule,
minai621 marked this conversation as resolved.
Show resolved Hide resolved
],
controllers: [],
providers: [
Expand Down
4 changes: 3 additions & 1 deletion backend/src/files/files.module.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import { Module } from "@nestjs/common";
import { PrismaService } from "src/db/prisma.service";
import { StorageModule } from "src/storage/storage.module";
import { FilesController } from "./files.controller";
import { FilesService } from "./files.service";
import { PrismaService } from "src/db/prisma.service";

@Module({
imports: [StorageModule],
controllers: [FilesController],
providers: [FilesService, PrismaService],
})
Expand Down
34 changes: 29 additions & 5 deletions backend/src/files/files.service.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
import { GetObjectCommand, PutObjectCommand, S3Client, S3ClientConfig } from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
import {
BadRequestException,
Inject,
Injectable,
NotFoundException,
UnauthorizedException,
Expand All @@ -13,19 +14,19 @@ import * as htmlPdf from "html-pdf-node";
import * as MarkdownIt from "markdown-it";
import { PrismaService } from "src/db/prisma.service";
import { generateRandomKey } from "src/utils/functions/random-string";
import { StorageType } from "src/utils/types/storage.type";
import { CreateUploadPresignedUrlResponse } from "./types/create-upload-url-response.type";
import { ExportFileRequestBody, ExportFileResponse } from "./types/export-file.type";

@Injectable()
export class FilesService {
private s3Client: S3Client;
private readonly markdown: MarkdownIt;

constructor(
@Inject("STORAGE_CLIENT") private s3Client: S3Client,
private configService: ConfigService,
private prismaService: PrismaService
) {
this.s3Client = new S3Client();
this.markdown = new MarkdownIt({
html: true,
breaks: true,
Expand Down Expand Up @@ -55,7 +56,7 @@ export class FilesService {

const fileKey = `${workspace.slug}-${generateRandomKey()}.${contentType.split("/")[1]}`;
const command = new PutObjectCommand({
Bucket: this.configService.get("AWS_S3_BUCKET_NAME"),
Bucket: this.configService.get("BUCKET_NAME"),
devleejb marked this conversation as resolved.
Show resolved Hide resolved
Key: fileKey,
StorageClass: "INTELLIGENT_TIERING",
ContentType: contentType,
Expand All @@ -70,7 +71,7 @@ export class FilesService {
async createDownloadPresignedUrl(fileKey: string) {
try {
const command = new GetObjectCommand({
Bucket: this.configService.get("AWS_S3_BUCKET_NAME"),
Bucket: this.configService.get("BUCKET_NAME"),
Key: fileKey,
});
return getSignedUrl(this.s3Client, command, { expiresIn: 3600 });
Expand Down Expand Up @@ -125,4 +126,27 @@ export class FilesService {
fileName: `${fileName}.pdf`,
};
}

private getStorageConfig = (): S3ClientConfig => {
const bucketType: StorageType = this.configService.get("BUCKET_TYPE") || "S3";
const region = this.configService.get("AWS_REGION") || "us-east-1";
if (bucketType === "MINIO") {
const endpoint = this.configService.get("MINIO_ENDPOINT");
const accessKeyId = this.configService.get("MINIO_ACCESS_KEY");
const secretAccessKey = this.configService.get("MINIO_SECRET_KEY");
return {
region,
endpoint,
forcePathStyle: true,
credentials: {
accessKeyId,
secretAccessKey,
},
};
}

return {
region,
};
};
devleejb marked this conversation as resolved.
Show resolved Hide resolved
}
2 changes: 1 addition & 1 deletion backend/src/settings/settings.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ export class SettingsService {
},
},
fileUpload: {
enable: this.configService.get("FILE_UPLOAD") === "true",
enable: this.configService.get("FILE_UPLOAD") !== "false",
devleejb marked this conversation as resolved.
Show resolved Hide resolved
},
};
}
Expand Down
39 changes: 39 additions & 0 deletions backend/src/storage/storage.module.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import { S3Client, S3ClientConfig } from "@aws-sdk/client-s3";
import { Module } from "@nestjs/common";
import { ConfigService } from "@nestjs/config";

const s3ClientFactory = {
provide: "STORAGE_CLIENT",
useFactory: (configService: ConfigService): S3Client | null => {
const fileUpload = configService.get<"false" | "s3" | "minio">("FILE_UPLOAD");
if (fileUpload === "false") {
return null;
}

const region = configService.get<string>("AWS_REGION");
const endpoint = configService.get<string>("MINIO_ENDPOINT");
const accessKeyId = configService.get<string>("MINIO_ACCESS_KEY");
const secretAccessKey = configService.get<string>("MINIO_SECRET_KEY");

const config: S3ClientConfig = {
region,
...(fileUpload === "minio" && {
endpoint,
forcePathStyle: true,
credentials: {
accessKeyId,
secretAccessKey,
},
}),
};

return new S3Client(config);
},
inject: [ConfigService],
};

@Module({
providers: [s3ClientFactory],
exports: [s3ClientFactory],
})
export class StorageModule {}
1 change: 1 addition & 0 deletions backend/src/utils/types/storage.type.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
export type StorageType = "S3" | "MINIO";
Loading