Skip to content

MINOR: Add check in ShareConsumerImpl to send acknowledgements of control records when ShareFetch is empty. #19295

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Original file line number Diff line number Diff line change
Expand Up @@ -256,13 +256,19 @@ public PollResult poll(long currentTimeMs) {
return new PollResult(requests);
}

public void fetch(Map<TopicIdPartition, NodeAcknowledgements> acknowledgementsMap) {
public void fetch(Map<TopicIdPartition, NodeAcknowledgements> acknowledgementsMap,
Map<TopicIdPartition, NodeAcknowledgements> controlRecordAcknowledgements) {
if (!fetchMoreRecords) {
log.debug("Fetch more data");
fetchMoreRecords = true;
}

// The acknowledgements sent via ShareFetch are stored in this map.
// Process both acknowledgement maps and sends them in the next ShareFetch.
processAcknowledgementsMap(acknowledgementsMap);
processAcknowledgementsMap(controlRecordAcknowledgements);
}

private void processAcknowledgementsMap(Map<TopicIdPartition, NodeAcknowledgements> acknowledgementsMap) {
acknowledgementsMap.forEach((tip, nodeAcks) -> {
int nodeId = nodeAcks.nodeId();
Map<TopicIdPartition, Acknowledgements> currentNodeAcknowledgementsMap = fetchAcknowledgementsToSend.get(nodeId);
Expand Down Expand Up @@ -1474,4 +1480,8 @@ public String toString() {
return super.toString().toLowerCase(Locale.ROOT);
}
}

Map<TopicIdPartition, Acknowledgements> getFetchAcknowledgementsToSend(Integer nodeId) {
return fetchAcknowledgementsToSend.get(nodeId);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -644,8 +644,8 @@ private ShareFetch<K, V> collect(Map<TopicIdPartition, NodeAcknowledgements> ack
if (currentFetch.isEmpty()) {
final ShareFetch<K, V> fetch = fetchCollector.collect(fetchBuffer);
if (fetch.isEmpty()) {
// Fetch more records and send any waiting acknowledgements
applicationEventHandler.add(new ShareFetchEvent(acknowledgementsMap));
// Check for any acknowledgements which could have come from control records (GAP) and include them.
applicationEventHandler.add(new ShareFetchEvent(acknowledgementsMap, fetch.takeAcknowledgedRecords()));

// Notify the network thread to wake up and start the next round of fetching
applicationEventHandler.wakeupNetworkThread();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,7 @@ private void process(@SuppressWarnings("unused") final StopFindCoordinatorOnClos
* Process event that tells the share consume request manager to fetch more records.
*/
private void process(final ShareFetchEvent event) {
requestManagers.shareConsumeRequestManager.ifPresent(scrm -> scrm.fetch(event.acknowledgementsMap()));
requestManagers.shareConsumeRequestManager.ifPresent(scrm -> scrm.fetch(event.acknowledgementsMap(), event.controlRecordAcknowledgements()));
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,23 @@ public class ShareFetchEvent extends ApplicationEvent {

private final Map<TopicIdPartition, NodeAcknowledgements> acknowledgementsMap;

public ShareFetchEvent(Map<TopicIdPartition, NodeAcknowledgements> acknowledgementsMap) {
private final Map<TopicIdPartition, NodeAcknowledgements> controlRecordAcknowledgements;

public ShareFetchEvent(Map<TopicIdPartition, NodeAcknowledgements> acknowledgementsMap,
Map<TopicIdPartition, NodeAcknowledgements> controlRecordAcknowledgements) {
super(Type.SHARE_FETCH);
this.acknowledgementsMap = acknowledgementsMap;
this.controlRecordAcknowledgements = controlRecordAcknowledgements;
}

public Map<TopicIdPartition, NodeAcknowledgements> acknowledgementsMap() {
return acknowledgementsMap;
}

public Map<TopicIdPartition, NodeAcknowledgements> controlRecordAcknowledgements() {
return controlRecordAcknowledgements;
}

@Override
protected String toStringBase() {
return super.toStringBase() + ", acknowledgementsMap=" + acknowledgementsMap;
Expand Down
Loading