Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(NODE-4719): Add SDAM Logging Spec #3940

Merged
merged 23 commits into from
Dec 14, 2023
Merged
Show file tree
Hide file tree
Changes from 21 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
5af76dd
added tests
aditi-khare-mongoDB Nov 30, 2023
79d23ed
Basic changes finished
aditi-khare-mongoDB Dec 4, 2023
5a5f612
Linting errors gone
aditi-khare-mongoDB Dec 4, 2023
1020dfa
spec runner changes working, may need to add filter to logs
aditi-khare-mongoDB Dec 5, 2023
f2c9e84
spec runner changes working, may need to add filter to logs
aditi-khare-mongoDB Dec 5, 2023
72a99ba
for evergreen patch
aditi-khare-mongoDB Dec 5, 2023
2b2fcb5
Filtering done! ready for PR
aditi-khare-mongoDB Dec 6, 2023
b71f1eb
Fixed failing unit test
aditi-khare-mongoDB Dec 7, 2023
be7aee9
PR requested changes
aditi-khare-mongoDB Dec 7, 2023
4cab947
lint fix
aditi-khare-mongoDB Dec 7, 2023
b1e2d42
fixed unit tests
aditi-khare-mongoDB Dec 7, 2023
61baff7
rebase
aditi-khare-mongoDB Dec 8, 2023
a240724
Added driverConnectionId and emitAndLogHeartbeat() function
aditi-khare-mongoDB Dec 8, 2023
00ab295
Lint fixes
aditi-khare-mongoDB Dec 8, 2023
c0e9159
Lint fix
aditi-khare-mongoDB Dec 11, 2023
69c05b1
PR requested changes
aditi-khare-mongoDB Dec 11, 2023
3f0ac27
passing all spec tests and integration tests
aditi-khare-mongoDB Dec 11, 2023
6cd5830
PR requested changes 2
aditi-khare-mongoDB Dec 11, 2023
f778cd0
Added comment '
aditi-khare-mongoDB Dec 11, 2023
add5421
PR requested chanes part 3
aditi-khare-mongoDB Dec 12, 2023
3576be7
evergreen failure fixes
aditi-khare-mongoDB Dec 12, 2023
1d6e837
emitAndLogHeartbeat is marked as internal now
aditi-khare-mongoDB Dec 12, 2023
a9c8d77
Merge branch 'main' into NODE-4719/SDAM-logging
nbbeeken Dec 13, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,17 @@ export const MESSAGE = 'message' as const;
export const PINNED = 'pinned' as const;
export const UNPINNED = 'unpinned' as const;
export const DESCRIPTION_RECEIVED = 'descriptionReceived';
/** @internal */
export const SERVER_OPENING = 'serverOpening' as const;
/** @internal */
export const SERVER_CLOSED = 'serverClosed' as const;
/** @internal */
export const SERVER_DESCRIPTION_CHANGED = 'serverDescriptionChanged' as const;
/** @internal */
export const TOPOLOGY_OPENING = 'topologyOpening' as const;
/** @internal */
export const TOPOLOGY_CLOSED = 'topologyClosed' as const;
/** @internal */
export const TOPOLOGY_DESCRIPTION_CHANGED = 'topologyDescriptionChanged' as const;
/** @internal */
export const CONNECTION_POOL_CREATED = 'connectionPoolCreated' as const;
Expand Down Expand Up @@ -49,8 +55,11 @@ export const CLUSTER_TIME_RECEIVED = 'clusterTimeReceived' as const;
export const COMMAND_STARTED = 'commandStarted' as const;
export const COMMAND_SUCCEEDED = 'commandSucceeded' as const;
export const COMMAND_FAILED = 'commandFailed' as const;
/** @internal */
export const SERVER_HEARTBEAT_STARTED = 'serverHeartbeatStarted' as const;
/** @internal */
export const SERVER_HEARTBEAT_SUCCEEDED = 'serverHeartbeatSucceeded' as const;
/** @internal */
export const SERVER_HEARTBEAT_FAILED = 'serverHeartbeatFailed' as const;
export const RESPONSE = 'response' as const;
export const MORE = 'more' as const;
Expand Down
14 changes: 13 additions & 1 deletion src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,16 @@ export type {
CONNECTION_POOL_CREATED,
CONNECTION_POOL_READY,
CONNECTION_READY,
MONGO_CLIENT_EVENTS
MONGO_CLIENT_EVENTS,
SERVER_CLOSED,
SERVER_DESCRIPTION_CHANGED,
SERVER_HEARTBEAT_FAILED,
SERVER_HEARTBEAT_STARTED,
SERVER_HEARTBEAT_SUCCEEDED,
SERVER_OPENING,
TOPOLOGY_CLOSED,
TOPOLOGY_DESCRIPTION_CHANGED,
TOPOLOGY_OPENING
} from './constants';
export type {
AbstractCursorEvents,
Expand Down Expand Up @@ -357,6 +366,9 @@ export type {
LogConvertible,
Loggable,
LoggableEvent,
LoggableServerHeartbeatFailedEvent,
LoggableServerHeartbeatStartedEvent,
LoggableServerHeartbeatSucceededEvent,
MongoDBLogWritable,
MongoLoggableComponent,
MongoLogger,
Expand Down
161 changes: 153 additions & 8 deletions src/mongo_logger.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { EJSON } from 'bson';
import { type Document, EJSON, type EJSONOptions } from 'bson';
import type { Writable } from 'stream';
import { inspect } from 'util';

Expand Down Expand Up @@ -35,8 +35,23 @@ import {
CONNECTION_POOL_CLOSED,
CONNECTION_POOL_CREATED,
CONNECTION_POOL_READY,
CONNECTION_READY
CONNECTION_READY,
SERVER_CLOSED,
SERVER_HEARTBEAT_FAILED,
SERVER_HEARTBEAT_STARTED,
SERVER_HEARTBEAT_SUCCEEDED,
SERVER_OPENING,
TOPOLOGY_CLOSED,
TOPOLOGY_DESCRIPTION_CHANGED,
TOPOLOGY_OPENING
} from './constants';
import type {
ServerClosedEvent,
ServerOpeningEvent,
TopologyClosedEvent,
TopologyDescriptionChangedEvent,
TopologyOpeningEvent
} from './sdam/events';
import { HostAddress, parseUnsignedInteger } from './utils';

/** @internal */
Expand Down Expand Up @@ -270,6 +285,54 @@ function compareSeverity(s0: SeverityLevel, s1: SeverityLevel): 1 | 0 | -1 {
return s0Num < s1Num ? -1 : s0Num > s1Num ? 1 : 0;
}

/**
* @internal
* Must be separate from Events API due to differences in spec requirements for logging server heartbeat beginning
*/
export type LoggableServerHeartbeatStartedEvent = {
topologyId: number;
awaited: boolean;
connectionId: string;
name: typeof SERVER_HEARTBEAT_STARTED;
};

/**
* @internal
* Must be separate from Events API due to differences in spec requirements for logging server heartbeat success
*/
export type LoggableServerHeartbeatSucceededEvent = {
topologyId: number;
awaited: boolean;
connectionId: string;
reply: Document;
serverConnectionId: number | '<monitor>';
duration: number;
name: typeof SERVER_HEARTBEAT_SUCCEEDED;
};

/**
* @internal
* Must be separate from Events API due to differences in spec requirements for logging server heartbeat failure
*/
export type LoggableServerHeartbeatFailedEvent = {
topologyId: number;
awaited: boolean;
connectionId: string;
failure: Error;
duration: number;
name: typeof SERVER_HEARTBEAT_FAILED;
};

type SDAMLoggableEvent =
| ServerClosedEvent
| LoggableServerHeartbeatFailedEvent
| LoggableServerHeartbeatStartedEvent
| LoggableServerHeartbeatSucceededEvent
| ServerOpeningEvent
| TopologyClosedEvent
| TopologyDescriptionChangedEvent
| TopologyOpeningEvent;

/** @internal */
export type LoggableEvent =
| CommandStartedEvent
Expand All @@ -285,16 +348,28 @@ export type LoggableEvent =
| ConnectionCheckedInEvent
| ConnectionCheckedOutEvent
| ConnectionCheckOutStartedEvent
| ConnectionCheckOutFailedEvent;
| ConnectionCheckOutFailedEvent
| ServerClosedEvent
| LoggableServerHeartbeatFailedEvent
| LoggableServerHeartbeatStartedEvent
| LoggableServerHeartbeatSucceededEvent
| ServerOpeningEvent
| TopologyClosedEvent
| TopologyDescriptionChangedEvent
| TopologyOpeningEvent;

/** @internal */
export interface LogConvertible extends Record<string, any> {
toLog(): Record<string, any>;
}

/** @internal */
export function stringifyWithMaxLen(value: any, maxDocumentLength: number): string {
const ejson = EJSON.stringify(value);
export function stringifyWithMaxLen(
value: any,
maxDocumentLength: number,
options: EJSONOptions = {}
): string {
const ejson = EJSON.stringify(value, options);

return maxDocumentLength !== 0 && ejson.length > maxDocumentLength
? `${ejson.slice(0, maxDocumentLength)}...`
Expand Down Expand Up @@ -329,15 +404,36 @@ function attachCommandFields(

function attachConnectionFields(
log: Record<string, any>,
connectionPoolEvent: ConnectionPoolMonitoringEvent
event: ConnectionPoolMonitoringEvent | ServerOpeningEvent | ServerClosedEvent
) {
const { host, port } = HostAddress.fromString(connectionPoolEvent.address).toHostPort();
const { host, port } = HostAddress.fromString(event.address).toHostPort();
log.serverHost = host;
log.serverPort = port;

return log;
}

function attachSDAMFields(log: Record<string, any>, sdamEvent: SDAMLoggableEvent) {
log.topologyId = sdamEvent.topologyId;
return log;
}

function attachServerHeartbeatFields(
log: Record<string, any>,
serverHeartbeatEvent:
| LoggableServerHeartbeatFailedEvent
| LoggableServerHeartbeatStartedEvent
| LoggableServerHeartbeatSucceededEvent
) {
const { awaited, connectionId } = serverHeartbeatEvent;
log.awaited = awaited;
log.driverConnectionId = serverHeartbeatEvent.connectionId;
const { host, port } = HostAddress.fromString(connectionId).toHostPort();
log.serverHost = host;
log.serverPort = port;
return log;
}

function defaultLogTransform(
logObject: LoggableEvent | Record<string, any>,
maxDocumentLength: number = DEFAULT_MAX_DOCUMENT_LENGTH
Expand Down Expand Up @@ -456,14 +552,63 @@ function defaultLogTransform(
case CONNECTION_CHECKED_OUT:
log = attachConnectionFields(log, logObject);
log.message = 'Connection checked out';

log.driverConnectionId = logObject.connectionId;
return log;
case CONNECTION_CHECKED_IN:
log = attachConnectionFields(log, logObject);
log.message = 'Connection checked in';
log.driverConnectionId = logObject.connectionId;
return log;
case SERVER_OPENING:
log = attachSDAMFields(log, logObject);
log = attachConnectionFields(log, logObject);
log.message = 'Starting server monitoring';
return log;
case SERVER_CLOSED:
log = attachSDAMFields(log, logObject);
log = attachConnectionFields(log, logObject);
log.message = 'Stopped server monitoring';
return log;
case SERVER_HEARTBEAT_STARTED:
log = attachSDAMFields(log, logObject);
log = attachServerHeartbeatFields(log, logObject);
log.message = 'Server heartbeat started';
return log;
case SERVER_HEARTBEAT_SUCCEEDED:
log = attachSDAMFields(log, logObject);
log = attachServerHeartbeatFields(log, logObject);
log.message = 'Server heartbeat succeeded';
log.durationMS = logObject.duration;
log.serverConnectionId = logObject.serverConnectionId;
log.reply = stringifyWithMaxLen(logObject.reply, maxDocumentLength, { relaxed: true });
return log;
case SERVER_HEARTBEAT_FAILED:
log = attachSDAMFields(log, logObject);
log = attachServerHeartbeatFields(log, logObject);
log.message = 'Server heartbeat failed';
log.durationMS = logObject.duration;
log.failure = logObject.failure.message;
return log;
case TOPOLOGY_OPENING:
log = attachSDAMFields(log, logObject);
log.message = 'Starting topology monitoring';
return log;
case TOPOLOGY_CLOSED:
log = attachSDAMFields(log, logObject);
log.message = 'Stopped topology monitoring';
return log;
case TOPOLOGY_DESCRIPTION_CHANGED:
log = attachSDAMFields(log, logObject);
log.message = 'Topology description changed';
log.previousDescription = log.reply = stringifyWithMaxLen(
logObject.previousDescription,
maxDocumentLength
);
log.newDescription = log.reply = stringifyWithMaxLen(
logObject.newDescription,
maxDocumentLength
);
return log;
default:
for (const [key, value] of Object.entries(logObject)) {
if (value != null) log[key] = value;
Expand Down
29 changes: 27 additions & 2 deletions src/mongo_types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,13 @@ import type {
ObjectId,
Timestamp
} from './bson';
import type { MongoLoggableComponent, MongoLogger } from './mongo_logger';
import type {
LoggableServerHeartbeatFailedEvent,
LoggableServerHeartbeatStartedEvent,
LoggableServerHeartbeatSucceededEvent,
MongoLoggableComponent,
MongoLogger
} from './mongo_logger';
import type { Sort } from './sort';

/** @internal */
Expand Down Expand Up @@ -405,13 +411,32 @@ export class TypedEventEmitter<Events extends EventsDescription> extends EventEm
/** @internal */
protected component?: MongoLoggableComponent;
/** @internal */
protected emitAndLog<EventKey extends keyof Events>(
emitAndLog<EventKey extends keyof Events>(
nbbeeken marked this conversation as resolved.
Show resolved Hide resolved
event: EventKey | symbol,
...args: Parameters<Events[EventKey]>
): void {
this.emit(event, ...args);
if (this.component) this.mongoLogger?.debug(this.component, args[0]);
}
emitAndLogHeartbeat<EventKey extends keyof Events>(
nbbeeken marked this conversation as resolved.
Show resolved Hide resolved
event: EventKey | symbol,
topologyId: number,
serverConnectionId?: number | '<monitor>',
...args: Parameters<Events[EventKey]>
): void {
this.emit(event, ...args);
if (this.component) {
const loggableHeartbeatEvent:
| LoggableServerHeartbeatFailedEvent
| LoggableServerHeartbeatSucceededEvent
| LoggableServerHeartbeatStartedEvent = {
topologyId: topologyId,
serverConnectionId: serverConnectionId ?? null,
...args[0]
};
this.mongoLogger?.debug(this.component, loggableHeartbeatEvent);
}
}
}

/** @public */
Expand Down
Loading