1// FetchRequest is a long-poll request of records from Kafka.
2//
3// Kafka 0.11.0.0 released v4 and changed the returned RecordBatches to contain
4// the RecordBatch type. Prior, Kafka used the MessageSet type (and, for v0 and
5// v1, Kafka used a different type).
6//
7// Note that starting in v3, Kafka began processing partitions in order,
8// meaning the order of partitions in the fetch request is important due to
9// potential size constraints.
10//
11// Starting in v13, topics must use UUIDs rather than their string name
12// identifiers.
13//
14// Version 15 adds the ReplicaState which includes new field ReplicaEpoch and
15// the ReplicaID, and deprecates the old ReplicaID (KIP-903).
16FetchRequest => key 1, max version 15, flexible v12+
17 // The cluster ID, if known. This is used to validate metadata fetches
18 // prior to broker registration.
19 ClusterID: nullable-string(null) // tag 0
20 // ReplicaID is the broker ID of performing the fetch request. Standard
21 // clients should use -1. To be a "debug" replica, use -2. The debug
22 // replica can be used to fetch messages from non-leaders.
23 ReplicaID: int32(-1) // v0-v14
24 // ReplicaState is a broker-only tag for v15+, see KIP-903 for more details.
25 ReplicaState: => // tag 1
26 // The replica ID of the follower, or -1 if this request is from a consumer.
27 ID: int32(-1)
28 // The epoch of this follower, or -1 if not available.
29 Epoch: int64(-1)
30 // MaxWaitMillis is how long to wait for MinBytes to be hit before a broker
31 // responds to a fetch request.
32 MaxWaitMillis: int32
33 // MinBytes is the minimum amount of bytes to attempt to read before a broker
34 // responds to a fetch request.
35 MinBytes: int32
36 // MaxBytes is the maximum amount of bytes to read in a fetch request. The
37 // response can exceed MaxBytes if the first record in the first non-empty
38 // partition is larger than MaxBytes.
39 MaxBytes: int32(0x7fffffff) // v3+
40 // IsolationLevel changes which messages are fetched. Follower replica ID's
41 // (non-negative, non-standard-client) fetch from the end.
42 //
43 // Standard clients fetch from the high watermark, which corresponds to
44 // IsolationLevel 0, READ_UNCOMMITTED.
45 //
46 // To only read committed records, use IsolationLevel 1, corresponding to
47 // READ_COMMITTED.
48 IsolationLevel: int8 // v4+
49 // SessionID is used to potentially reduce the amount of back and forth
50 // data between a client and a broker. If opting in to sessions, the first
51 // ID used should be 0, and thereafter (until session resets) the ID should
52 // be the ID returned in the fetch response.
53 //
54 // Read KIP-227 for more details. Use -1 if you want to disable sessions.
55 SessionID: int32 // v7+
56 // SessionEpoch is the session epoch for this request if using sessions.
57 //
58 // Read KIP-227 for more details. Use -1 if you are not using sessions.
59 SessionEpoch: int32(-1) // v7+
60 // Topic contains topics to try to fetch records for.
61 Topics: [=>]
62 // Topic is a topic to try to fetch records for.
63 Topic: string // v0-v12
64 // TopicID is the uuid of the topic to fetch records for.
65 TopicID: uuid // v13+
66 // Partitions contains partitions in a topic to try to fetch records for.
67 Partitions: [=>]
68 // Partition is a partition in a topic to try to fetch records for.
69 Partition: int32
70 // CurrentLeaderEpoch, proposed in KIP-320 and introduced in Kafka 2.1.0,
71 // allows brokers to check if the client is fenced (has an out of date
72 // leader) or is using an unknown leader.
73 //
74 // The initial leader epoch can be determined from a MetadataResponse.
75 // To skip log truncation checking, use -1.
76 CurrentLeaderEpoch: int32(-1) // v9+
77 // FetchOffset is the offset to begin the fetch from. Kafka will
78 // return records at and after this offset.
79 FetchOffset: int64
80 // The epoch of the last fetched record, or -1 if there is none.
81 LastFetchedEpoch: int32(-1) // v12+
82 // LogStartOffset is a broker-follower only field added for KIP-107.
83 // This is the start offset of the partition in a follower.
84 LogStartOffset: int64(-1) // v5+
85 // PartitionMaxBytes is the maximum bytes to return for this partition.
86 // This can be used to limit how many bytes an individual partition in
87 // a request is allotted so that it does not dominate all of MaxBytes.
88 PartitionMaxBytes: int32
89 // ForgottenTopics contains topics and partitions that a fetch session
90 // wants to remove from its session.
91 //
92 // See KIP-227 for more details.
93 ForgottenTopics: [=>] // v7+
94 // Topic is a topic to remove from being tracked (with the partitions below).
95 Topic: string // v7-v12
96 // TopicID is the uuid of a topic to remove from being tracked (with the
97 // partitions below).
98 TopicID: uuid // v13+
99 // Partitions are partitions to remove from tracking for a topic.
100 Partitions: [int32]
101 // Rack of the consumer making this request (see KIP-392; introduced in
102 // Kafka 2.2.0).
103 Rack: string // v11+
104
105// FetchResponse is returned from a FetchRequest.
106FetchResponse =>
107 ThrottleMillis(8) // v1+
108 // ErrorCode is a full-response error code for a fetch request. This was
109 // added in support of KIP-227. This error is only non-zero if using fetch
110 // sessions.
111 //
112 // FETCH_SESSION_ID_NOT_FOUND is returned if the request used a
113 // session ID that the broker does not know of.
114 //
115 // INVALID_FETCH_SESSION_EPOCH is returned if the request used an
116 // invalid session epoch.
117 ErrorCode: int16 // v7+
118 // SessionID is the id for this session if using sessions.
119 //
120 // See KIP-227 for more details.
121 SessionID: int32 // v7+
122 // Topics contains an array of topic partitions and the records received
123 // for them.
124 Topics: [=>]
125 // Topic is a topic that records may have been received for.
126 Topic: string // v0-v12
127 // TopicID is the uuid of a topic that records may have been received for.
128 TopicID: uuid // v13+
129 // Partitions contains partitions in a topic that records may have
130 // been received for.
131 Partitions: [=>]
132 // Partition is a partition in a topic that records may have been
133 // received for.
134 Partition: int32
135 // ErrorCode is an error returned for an individual partition in a
136 // fetch request.
137 //
138 // TOPIC_AUTHORIZATION_FAILED is returned if the client is not
139 // authorized to read the partition.
140 //
141 // UNKNOWN_TOPIC_OR_PARTITION is returned if the topic or partition
142 // does not exist on this broker.
143 //
144 // UNSUPPORTED_COMPRESSION_TYPE is returned if the request version was
145 // under 10 and the batch is compressed with zstd.
146 //
147 // UNSUPPORTED_VERSION is returned if the broker has records newer than
148 // the client can support (magic value) and the broker has disabled
149 // message downconversion.
150 //
151 // NOT_LEADER_FOR_PARTITION is returned if requesting data for this
152 // partition as a follower (non-negative ReplicaID) and the broker
153 // is not the leader for this partition.
154 //
155 // REPLICA_NOT_AVAILABLE is returned if the partition exists but
156 // the requested broker is not the leader for it.
157 //
158 // KAFKA_STORAGE_EXCEPTION is returned if the requested partition is
159 // offline.
160 //
161 // UNKNOWN_LEADER_EPOCH is returned if the request used a larger leader
162 // epoch than the broker knows of.
163 //
164 // FENCED_LEADER_EPOCH is returned if the request used a smaller leader
165 // epoch than the broker is at (see KIP-320).
166 //
167 // OFFSET_OUT_OF_RANGE is returned if requesting an offset past the
168 // current end offset or before the beginning offset.
169 //
170 // UNKNOWN_TOPIC_ID is returned if using uuid's and the uuid is unknown
171 // (v13+ / Kafka 3.1+).
172 //
173 // OFFSET_MOVED_TO_TIERED_STORAGE is returned if a follower is trying to
174 // fetch from an offset that is now in tiered storage.
175 ErrorCode: int16
176 // HighWatermark is the current high watermark for this partition,
177 // that is, the current offset that is on all in sync replicas.
178 HighWatermark: int64
179 // LastStableOffset is the offset at which all prior offsets have
180 // been "decided". Non transactional records are always decided
181 // immediately, but transactional records are only decided once
182 // they are commited or aborted.
183 //
184 // The LastStableOffset will always be at or under the HighWatermark.
185 LastStableOffset: int64(-1) // v4+
186 // LogStartOffset is the beginning offset for this partition.
187 // This field was added for KIP-107.
188 LogStartOffset: int64(-1) // v5+
189 // In case divergence is detected based on the LastFetchedEpoch and
190 // FetchOffset in the request, this field indicates the largest epoch and
191 // its end offset such that subsequent records are known to diverge.
192 DivergingEpoch: => // tag 0
193 Epoch: int32(-1)
194 EndOffset: int64(-1)
195 // CurrentLeader is the currently known leader ID and epoch for this
196 // partition.
197 CurrentLeader: => // tag 1
198 // The ID of the current leader, or -1 if unknown.
199 LeaderID: int32(-1)
200 // The latest known leader epoch.
201 LeaderEpoch: int32(-1)
202 // In the case of fetching an offset less than the LogStartOffset, this
203 // is the end offset and epoch that should be used in the FetchSnapshot
204 // request.
205 SnapshotID: => // tag 2
206 EndOffset: int64(-1)
207 Epoch: int32(-1)
208 // AbortedTransactions is an array of aborted transactions within the
209 // returned offset range. This is only returned if the requested
210 // isolation level was READ_COMMITTED.
211 AbortedTransactions: nullable[=>] // v4+
212 // ProducerID is the producer ID that caused this aborted transaction.
213 ProducerID: int64
214 // FirstOffset is the offset where this aborted transaction began.
215 FirstOffset: int64
216 // PreferredReadReplica is the preferred replica for the consumer
217 // to use on its next fetch request. See KIP-392.
218 PreferredReadReplica: int32(-1) // v11+
219 // RecordBatches is an array of record batches for a topic partition.
220 //
221 // This is encoded as a raw byte array, with the standard int32 size
222 // prefix. One important catch to note is that the final element of the
223 // array may be **partial**. This is an optimization in Kafka that
224 // clients must deal with by discarding a partial trailing batch.
225 //
226 // Starting v2, this transitioned to the MessageSet v1 format (and this
227 // would contain many MessageV1 structs).
228 //
229 // Starting v4, this transitioned to the RecordBatch format (thus this
230 // contains many RecordBatch structs).
231 RecordBatches: nullable-bytes
View as plain text