Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Freshdesk Connector #2884

Merged
merged 21 commits into from
Nov 2, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions backend/danswer/configs/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ class DocumentSource(str, Enum):
OCI_STORAGE = "oci_storage"
XENFORO = "xenforo"
NOT_APPLICABLE = "not_applicable"
FRESHDESK = "freshdesk"


DocumentSourceRequiringTenantContext: list[DocumentSource] = [DocumentSource.FILE]
Expand Down
2 changes: 2 additions & 0 deletions backend/danswer/connectors/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
from danswer.connectors.xenforo.connector import XenforoConnector
from danswer.connectors.zendesk.connector import ZendeskConnector
from danswer.connectors.zulip.connector import ZulipConnector
from danswer.connectors.freshdesk.connector import FreshdeskConnector
from danswer.db.credentials import backend_update_credential_json
from danswer.db.models import Credential

Expand Down Expand Up @@ -101,6 +102,7 @@ def identify_connector_class(
DocumentSource.GOOGLE_CLOUD_STORAGE: BlobStorageConnector,
DocumentSource.OCI_STORAGE: BlobStorageConnector,
DocumentSource.XENFORO: XenforoConnector,
DocumentSource.FRESHDESK: FreshdeskConnector,
}
connector_by_source = connector_map.get(source, {})

Expand Down
Empty file.
115 changes: 115 additions & 0 deletions backend/danswer/connectors/freshdesk/connector.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
import requests
import json
from datetime import datetime, timezone
from typing import Any, List, Optional
from bs4 import BeautifulSoup # Add this import for HTML parsing
from danswer.configs.app_configs import INDEX_BATCH_SIZE
from danswer.configs.constants import DocumentSource
from danswer.connectors.interfaces import GenerateDocumentsOutput, PollConnector
from danswer.connectors.models import ConnectorMissingCredentialError, Document, Section
from danswer.utils.logger import setup_logger

logger = setup_logger()


skylares marked this conversation as resolved.
Show resolved Hide resolved
class FreshdeskConnector(PollConnector):
def __init__(self, batch_size: int = INDEX_BATCH_SIZE) -> None:
self.batch_size = batch_size

def ticket_link(self, tid: int) -> str:
skylares marked this conversation as resolved.
Show resolved Hide resolved
return f"https://{self.domain}.freshdesk.com/helpdesk/tickets/{tid}"

def build_doc_sections_from_ticket(self, ticket: dict) -> List[Section]:
# Use list comprehension for building sections
skylares marked this conversation as resolved.
Show resolved Hide resolved
return [
Section(
link=self.ticket_link(int(ticket["id"])),
text=json.dumps({
key: value
for key, value in ticket.items()
if isinstance(value, str)
}, default=str),
)
]

skylares marked this conversation as resolved.
Show resolved Hide resolved
def strip_html_tags(self, html: str) -> str:
soup = BeautifulSoup(html, 'html.parser')
return soup.get_text()

def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, Any]]:
logger.info("Loading credentials")
skylares marked this conversation as resolved.
Show resolved Hide resolved
self.api_key = credentials.get("freshdesk_api_key")
self.domain = credentials.get("freshdesk_domain")
self.password = credentials.get("freshdesk_password")
return None

def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
skylares marked this conversation as resolved.
Show resolved Hide resolved
logger.info("Processing tickets")
skylares marked this conversation as resolved.
Show resolved Hide resolved
if any([self.api_key, self.domain, self.password]) is None:
skylares marked this conversation as resolved.
Show resolved Hide resolved
raise ConnectorMissingCredentialError("freshdesk")

skylares marked this conversation as resolved.
Show resolved Hide resolved
freshdesk_url = f"https://{self.domain}.freshdesk.com/api/v2/tickets?include=description"
response = requests.get(freshdesk_url, auth=(self.api_key, self.password))
response.raise_for_status() # raises exception when not a 2xx response

skylares marked this conversation as resolved.
Show resolved Hide resolved
if response.status_code!= 204:
tickets = json.loads(response.content)
logger.info(f"Fetched {len(tickets)} tickets from Freshdesk API")
doc_batch: List[Document] = []

for ticket in tickets:
# Convert the "created_at", "updated_at", and "due_by" values to ISO 8601 strings
for date_field in ["created_at", "updated_at", "due_by"]:
if ticket[date_field].endswith('Z'):
ticket[date_field] = ticket[date_field][:-1] + '+00:00'
ticket[date_field] = datetime.fromisoformat(ticket[date_field]).strftime("%Y-%m-%d %H:%M:%S")

# Convert all other values to strings
ticket = {
hagen-danswer marked this conversation as resolved.
Show resolved Hide resolved
key: str(value) if not isinstance(value, str) else value
for key, value in ticket.items()
}

# Checking for overdue tickets
today = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
ticket["overdue"] = "true" if today > ticket["due_by"] else "false"

# Mapping the status field values
status_mapping = {2: "open", 3: "pending", 4: "resolved", 5: "closed"}
ticket["status"] = status_mapping.get(ticket["status"], str(ticket["status"]))

# Stripping HTML tags from the description field
ticket["description"] = self.strip_html_tags(ticket["description"])

# Remove extra white spaces from the description field
ticket["description"] = " ".join(ticket["description"].split())

# Use list comprehension for building sections
sections = self.build_doc_sections_from_ticket(ticket)

created_at = datetime.fromisoformat(ticket["created_at"])
today = datetime.now()
if (today - created_at).days / 30.4375 <= 2:
skylares marked this conversation as resolved.
Show resolved Hide resolved
doc = Document(
id=ticket["id"],
sections=sections,
source=DocumentSource.FRESHDESK,
semantic_identifier=ticket["subject"],
metadata={
hagen-danswer marked this conversation as resolved.
Show resolved Hide resolved
key: value
for key, value in ticket.items()
if isinstance(value, str) and key not in ["description", "description_text"]
},
)

doc_batch.append(doc)

if len(doc_batch) >= self.batch_size:
yield doc_batch
doc_batch = []

if doc_batch:
yield doc_batch

def poll_source(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
yield from self._process_tickets(start, end)
skylares marked this conversation as resolved.
Show resolved Hide resolved
2 changes: 1 addition & 1 deletion backend/danswer/db/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from enum import Enum as PyEnum
from typing import Any
from typing import Literal
from typing import NotRequired
from typing_extensions import NotRequired
from typing import Optional
from uuid import uuid4
from typing_extensions import TypedDict # noreorder
Expand Down
2 changes: 1 addition & 1 deletion backend/danswer/file_store/models.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import base64
from enum import Enum
from typing import NotRequired
from typing_extensions import NotRequired
from typing_extensions import TypedDict # noreorder

from pydantic import BaseModel
Expand Down
2 changes: 1 addition & 1 deletion backend/model_server/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"

HF_CACHE_PATH = Path("/root/.cache/huggingface/")
TEMP_HF_CACHE_PATH = Path("/root/.cache/temp_huggingface/")
TEMP_HF_CACHE_PATH = Path.home() / ".cache" / "temp_huggingface"

transformer_logging.set_verbosity_error()

Expand Down
Binary file added web/public/Freshdesk.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
6 changes: 5 additions & 1 deletion web/src/app/api/[...path]/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,13 @@ async function handleRequest(request: NextRequest, path: string[]) {
backendUrl.searchParams.append(key, value);
});

// Create a new headers object, omitting the 'connection' header
const headers = new Headers(request.headers);
headers.delete('connection');

const response = await fetch(backendUrl, {
method: request.method,
headers: request.headers,
headers: headers,
body: request.body,
signal: request.signal,
// @ts-ignore
Expand Down
8 changes: 8 additions & 0 deletions web/src/components/icons/icons.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ import slackIcon from "../../../public/Slack.png";
import s3Icon from "../../../public/S3.png";
import r2Icon from "../../../public/r2.png";
import salesforceIcon from "../../../public/Salesforce.png";
import freshdeskIcon from "../../../public/Freshdesk.png";

import sharepointIcon from "../../../public/Sharepoint.png";
import teamsIcon from "../../../public/Teams.png";
Expand Down Expand Up @@ -1301,6 +1302,13 @@ export const AsanaIcon = ({
className = defaultTailwindCSS,
}: IconProps) => <LogoIcon size={size} className={className} src={asanaIcon} />;

export const FreshdeskIcon = ({
size = 16,
className = defaultTailwindCSS,
}: IconProps) => (
<LogoIcon size={size} className={className} src={freshdeskIcon} />
);

/*
EE Icons
*/
Expand Down
11 changes: 11 additions & 0 deletions web/src/lib/connectors/connectors.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -922,6 +922,12 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
],
advanced_values: [],
},
freshdesk: {
description: "Configure Freshdesk connector",
values: [],
advanced_values: [],
},

};
export function createConnectorInitialValues(
connector: ConfigurableSources
Expand Down Expand Up @@ -1180,6 +1186,11 @@ export interface AsanaConfig {
asana_team_id?: string;
}

export interface FreshdeskConfig {
requested_objects?: string[];
skylares marked this conversation as resolved.
Show resolved Hide resolved
}


export interface MediaWikiConfig extends MediaWikiBaseConfig {
hostname: string;
}
Expand Down
16 changes: 16 additions & 0 deletions web/src/lib/connectors/credentials.ts
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,12 @@ export interface AxeroCredentialJson {
axero_api_token: string;
}

export interface FreshdeskCredentialJson {
freshdesk_domain: string;
freshdesk_password: string;
freshdesk_api_key: string;
}

export interface MediaWikiCredentialJson {}
export interface WikipediaCredentialJson extends MediaWikiCredentialJson {}

Expand Down Expand Up @@ -289,6 +295,11 @@ export const credentialTemplates: Record<ValidSources, any> = {
access_key_id: "",
secret_access_key: "",
} as OCICredentialJson,
freshdesk: {
freshdesk_domain: "",
freshdesk_password: "",
freshdesk_api_key: "",
} as FreshdeskCredentialJson,
xenforo: null,
google_sites: null,
file: null,
Expand Down Expand Up @@ -435,6 +446,11 @@ export const credentialDisplayNames: Record<string, string> = {
// Axero
base_url: "Axero Base URL",
axero_api_token: "Axero API Token",

// Freshdesk
freshdesk_domain: "Freshdesk Domain",
freshdesk_password: "Freshdesk Password",
freshdesk_api_key: "Freshdesk API Key",
};
export function getDisplayNameForCredentialKey(key: string): string {
return credentialDisplayNames[key] || key;
Expand Down
7 changes: 7 additions & 0 deletions web/src/lib/sources.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import {
GoogleStorageIcon,
ColorSlackIcon,
XenforoIcon,
FreshdeskIcon,
} from "@/components/icons/icons";
import { ValidSources } from "./types";
import {
Expand Down Expand Up @@ -289,6 +290,12 @@ const SOURCE_METADATA_MAP: SourceMap = {
displayName: "Ingestion",
category: SourceCategory.Other,
},
freshdesk: {
icon: FreshdeskIcon,
displayName: "Freshdesk",
category: SourceCategory.CustomerSupport,
docs: "https://docs.danswer.dev/connectors/freshdesk",
},
// currently used for the Internet Search tool docs, which is why
// a globe is used
not_applicable: {
Expand Down
1 change: 1 addition & 0 deletions web/src/lib/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,7 @@ const validSources = [
"oci_storage",
"not_applicable",
"ingestion_api",
"freshdesk",
] as const;

export type ValidSources = (typeof validSources)[number];
Expand Down
Loading