-
Notifications
You must be signed in to change notification settings - Fork 1.2k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[Event Hubs] Updates APIs surrounding event processing and checkpointing #4994
Changes from 4 commits
c2f70ad
7e7c57d
9c5f777
7b2fbda
5f0d28d
624ca28
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,43 +1,37 @@ | ||
import { | ||
EventHubClient, | ||
ReceivedEventData, | ||
EventPosition, | ||
delay, | ||
EventProcessor, | ||
PartitionContext, | ||
InMemoryPartitionManager, | ||
CheckpointManager | ||
PartitionProcessor, | ||
CloseReason | ||
} from "@azure/event-hubs"; | ||
|
||
class SimplePartitionProcessor { | ||
private _context: PartitionContext; | ||
private _checkpointManager: CheckpointManager; | ||
constructor(context: PartitionContext, checkpointManager: CheckpointManager) { | ||
this._context = context; | ||
this._checkpointManager = checkpointManager; | ||
} | ||
async processEvents(events: ReceivedEventData[]) { | ||
if(events.length === 0){ | ||
class SamplePartitionProcessor extends PartitionProcessor { | ||
private _messageCount = 0; | ||
|
||
async processEvents(events: ReceivedEventData[], partitionContext: PartitionContext) { | ||
if (events.length === 0) { | ||
return; | ||
} | ||
for (const event of events) { | ||
console.log( | ||
"Received event: '%s' from partition: '%s' and consumer group: '%s'", | ||
event.body, | ||
this._context.partitionId, | ||
this._context.consumerGroupName | ||
`Received event: '${event.body}' from partition: '${partitionContext.partitionId}' and consumer group: '${partitionContext.consumerGroupName}'`, | ||
); | ||
try { | ||
// checkpoint using the last event in the batch | ||
await this._checkpointManager.updateCheckpoint(events[events.length - 1]); | ||
await partitionContext.updateCheckpoint(events[events.length - 1]); | ||
this._messageCount++; | ||
console.log( | ||
"Successfully checkpointed event: '%s' from partition: '%s'", | ||
events[events.length - 1].body, | ||
this._context.partitionId | ||
partitionContext.partitionId | ||
); | ||
} catch (err) { | ||
console.log( | ||
`Encountered an error while checkpointing on ${this._context.partitionId}: ${err.message}` | ||
`Encountered an error while checkpointing on ${partitionContext.partitionId}: ${err.message}` | ||
); | ||
} | ||
} | ||
|
@@ -47,12 +41,13 @@ class SimplePartitionProcessor { | |
console.log(`Encountered an error: ${error.message}`); | ||
} | ||
|
||
async initialize() { | ||
console.log(`Started processing`); | ||
async initialize(partitionContext: PartitionContext) { | ||
console.log(`Started processing partition: ${partitionContext.partitionId}`); | ||
} | ||
|
||
async close() { | ||
console.log(`Stopped processing`); | ||
async close(reason: CloseReason, partitionContext: PartitionContext) { | ||
console.log(`Stopped processing for reason ${reason}`); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Will this print out the reason in string or number? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This prints out the reason as a string. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That said, I think we should probably change the |
||
console.log(`Processed ${this._messageCount} from partition ${partitionContext.partitionId}.`); | ||
} | ||
} | ||
|
||
|
@@ -63,17 +58,12 @@ const eventHubName = ""; | |
async function main() { | ||
const client = new EventHubClient(connectionString, eventHubName); | ||
|
||
const eventProcessorFactory = (context: PartitionContext, checkpoint: CheckpointManager) => { | ||
return new SimplePartitionProcessor(context, checkpoint); | ||
}; | ||
|
||
const processor = new EventProcessor( | ||
EventHubClient.defaultConsumerGroupName, | ||
client, | ||
eventProcessorFactory, | ||
SamplePartitionProcessor, | ||
new InMemoryPartitionManager(), | ||
{ | ||
initialEventPosition: EventPosition.earliest(), | ||
maxBatchSize: 10, | ||
maxWaitTimeInSeconds: 20 | ||
} | ||
|
This file was deleted.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why are we checkpointing inside the for loop?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Good point. I only looked at changing the API to use the new version, missed the fact that it was in the for loop this whole time. I'll update it.