|
559968
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ffdeb333-2136-413d-a00d-4427e3c445f0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ffdeb333-2136-413d-a00d-4427e3c445f0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:40
|
26 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279984.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279984.0 (TID 279984) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-af46a0c0-cf98-4c5d-a000-4a792bdb191c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279984.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279984.0 (TID 279984) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-af46a0c0-cf98-4c5d-a000-4a792bdb191c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559969
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ffdeb333-2136-413d-a00d-4427e3c445f0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ffdeb333-2136-413d-a00d-4427e3c445f0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:40
|
24 ms
|
|
[279984]
|
|
|
559376
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ff7bc162-ed09-4e63-9888-baedaeca8a1a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ff7bc162-ed09-4e63-9888-baedaeca8a1a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:28
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279688.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279688.0 (TID 279688) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-452968b1-5dbc-46ef-aaae-b3828f654d3f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279688.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279688.0 (TID 279688) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-452968b1-5dbc-46ef-aaae-b3828f654d3f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559377
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ff7bc162-ed09-4e63-9888-baedaeca8a1a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ff7bc162-ed09-4e63-9888-baedaeca8a1a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:28
|
11 ms
|
|
[279688]
|
|
|
559502
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ff756549-28f6-4f99-b96b-6b5865f5a67c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ff756549-28f6-4f99-b96b-6b5865f5a67c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:30
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279751.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279751.0 (TID 279751) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-150bd523-e41e-4fd1-bad4-c9a91107c4df-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279751.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279751.0 (TID 279751) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-150bd523-e41e-4fd1-bad4-c9a91107c4df-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559503
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ff756549-28f6-4f99-b96b-6b5865f5a67c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ff756549-28f6-4f99-b96b-6b5865f5a67c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:30
|
13 ms
|
|
[279751]
|
|
|
559830
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fefef856-d8c7-43fc-829c-348ac6e778b1
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fefef856-d8c7-43fc-829c-348ac6e778b1
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:38
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279915.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279915.0 (TID 279915) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f87efe1b-6606-4e01-920b-4920a3c6f0e0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279915.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279915.0 (TID 279915) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f87efe1b-6606-4e01-920b-4920a3c6f0e0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559831
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fefef856-d8c7-43fc-829c-348ac6e778b1
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fefef856-d8c7-43fc-829c-348ac6e778b1
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:38
|
11 ms
|
|
[279915]
|
|
|
559306
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fed80447-7e9a-4ab7-8759-f00a6af3fb28
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fed80447-7e9a-4ab7-8759-f00a6af3fb28
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:26
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279653.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279653.0 (TID 279653) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-eccfee1a-7dd7-4ed5-a42b-6211f73b1af1-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279653.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279653.0 (TID 279653) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-eccfee1a-7dd7-4ed5-a42b-6211f73b1af1-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559307
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fed80447-7e9a-4ab7-8759-f00a6af3fb28
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fed80447-7e9a-4ab7-8759-f00a6af3fb28
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:26
|
11 ms
|
|
[279653]
|
|
|
559124
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = feb5541e-1c61-4f4b-a11b-20686123743c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = feb5541e-1c61-4f4b-a11b-20686123743c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:22
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279562.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279562.0 (TID 279562) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-fcd2c3e5-bf74-40d9-adce-ed378682d1b0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279562.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279562.0 (TID 279562) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-fcd2c3e5-bf74-40d9-adce-ed378682d1b0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559125
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = feb5541e-1c61-4f4b-a11b-20686123743c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = feb5541e-1c61-4f4b-a11b-20686123743c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:22
|
11 ms
|
|
[279562]
|
|
|
559886
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fe54e0cb-8847-49fb-b6b1-914a4993b846
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fe54e0cb-8847-49fb-b6b1-914a4993b846
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:39
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279943.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279943.0 (TID 279943) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-e2b27c41-0747-4179-b62b-54b36b824a5b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279943.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279943.0 (TID 279943) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-e2b27c41-0747-4179-b62b-54b36b824a5b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559887
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fe54e0cb-8847-49fb-b6b1-914a4993b846
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fe54e0cb-8847-49fb-b6b1-914a4993b846
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:39
|
12 ms
|
|
[279943]
|
|
|
559926
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fdecf641-65cd-40e1-9273-3d8d019ccd6d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fdecf641-65cd-40e1-9273-3d8d019ccd6d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:40
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279963.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279963.0 (TID 279963) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-aa9324d0-6238-4994-85ba-2bc123cf040e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279963.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279963.0 (TID 279963) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-aa9324d0-6238-4994-85ba-2bc123cf040e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559927
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fdecf641-65cd-40e1-9273-3d8d019ccd6d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fdecf641-65cd-40e1-9273-3d8d019ccd6d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:40
|
11 ms
|
|
[279963]
|
|
|
559202
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fd14a190-100b-4f3d-bdb3-284612b94df9
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fd14a190-100b-4f3d-bdb3-284612b94df9
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:24
|
12 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279601.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279601.0 (TID 279601) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-db19456c-301d-4dd7-ba86-76cdb2cb02a7-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279601.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279601.0 (TID 279601) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-db19456c-301d-4dd7-ba86-76cdb2cb02a7-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559203
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fd14a190-100b-4f3d-bdb3-284612b94df9
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fd14a190-100b-4f3d-bdb3-284612b94df9
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:24
|
10 ms
|
|
[279601]
|
|
|
560026
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fca11cc8-f8f1-474d-90ba-982489a1189d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fca11cc8-f8f1-474d-90ba-982489a1189d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:42
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 280013.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280013.0 (TID 280013) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-30f295e6-619b-40b0-9664-5ce1ea8ebb22-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 280013.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280013.0 (TID 280013) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-30f295e6-619b-40b0-9664-5ce1ea8ebb22-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
560027
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fca11cc8-f8f1-474d-90ba-982489a1189d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fca11cc8-f8f1-474d-90ba-982489a1189d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:42
|
11 ms
|
|
[280013]
|
|
|
559200
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fc36cb1e-da16-42f9-a1dc-aae1a76f7398
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fc36cb1e-da16-42f9-a1dc-aae1a76f7398
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:24
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279600.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279600.0 (TID 279600) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-66521218-a48a-446b-870d-b5f549f33225-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279600.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279600.0 (TID 279600) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-66521218-a48a-446b-870d-b5f549f33225-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559201
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fc36cb1e-da16-42f9-a1dc-aae1a76f7398
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fc36cb1e-da16-42f9-a1dc-aae1a76f7398
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:24
|
11 ms
|
|
[279600]
|
|
|
559894
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fbfb699d-0e7a-427c-8d59-a9bdb6c5a908
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fbfb699d-0e7a-427c-8d59-a9bdb6c5a908
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:39
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279947.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279947.0 (TID 279947) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-e5622e34-e4f7-4211-b035-2bb7b05562d3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279947.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279947.0 (TID 279947) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-e5622e34-e4f7-4211-b035-2bb7b05562d3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559895
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fbfb699d-0e7a-427c-8d59-a9bdb6c5a908
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fbfb699d-0e7a-427c-8d59-a9bdb6c5a908
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:39
|
13 ms
|
|
[279947]
|
|
|
559520
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fb2618ef-6a2f-4ad6-8e35-054b38324a0a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fb2618ef-6a2f-4ad6-8e35-054b38324a0a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:31
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279760.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279760.0 (TID 279760) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-de430517-7bea-42de-991d-0c4abf23c628-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279760.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279760.0 (TID 279760) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-de430517-7bea-42de-991d-0c4abf23c628-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559521
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fb2618ef-6a2f-4ad6-8e35-054b38324a0a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fb2618ef-6a2f-4ad6-8e35-054b38324a0a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:31
|
13 ms
|
|
[279760]
|
|
|
559526
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f9b1c36e-d50c-41ac-907a-ff71ee47e439
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f9b1c36e-d50c-41ac-907a-ff71ee47e439
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:31
|
27 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279763.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279763.0 (TID 279763) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-87822c79-e16f-4446-a594-1cc6efd5853c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279763.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279763.0 (TID 279763) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-87822c79-e16f-4446-a594-1cc6efd5853c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559527
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f9b1c36e-d50c-41ac-907a-ff71ee47e439
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f9b1c36e-d50c-41ac-907a-ff71ee47e439
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:31
|
24 ms
|
|
[279763]
|
|
|
559892
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f93823bf-c857-418e-9370-83e99c3a4996
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f93823bf-c857-418e-9370-83e99c3a4996
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:39
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279946.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279946.0 (TID 279946) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-83f0d4c6-5b8b-4890-bcb6-82843a016012-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279946.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279946.0 (TID 279946) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-83f0d4c6-5b8b-4890-bcb6-82843a016012-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559893
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f93823bf-c857-418e-9370-83e99c3a4996
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f93823bf-c857-418e-9370-83e99c3a4996
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:39
|
12 ms
|
|
[279946]
|
|
|
559636
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f9323cbb-f2c2-4077-8576-2b6b805958c3
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f9323cbb-f2c2-4077-8576-2b6b805958c3
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:33
|
25 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279818.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279818.0 (TID 279818) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1b7cf437-5e47-4405-a93f-b202f181054e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279818.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279818.0 (TID 279818) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1b7cf437-5e47-4405-a93f-b202f181054e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559637
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f9323cbb-f2c2-4077-8576-2b6b805958c3
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f9323cbb-f2c2-4077-8576-2b6b805958c3
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:33
|
11 ms
|
|
[279818]
|
|
|
559644
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f8573f4c-305d-4a93-ac55-9c4373832380
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f8573f4c-305d-4a93-ac55-9c4373832380
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:34
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279822.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279822.0 (TID 279822) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9de06a69-135e-49e5-ae5b-3f7ae40beaa8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279822.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279822.0 (TID 279822) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9de06a69-135e-49e5-ae5b-3f7ae40beaa8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559645
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f8573f4c-305d-4a93-ac55-9c4373832380
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f8573f4c-305d-4a93-ac55-9c4373832380
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:34
|
10 ms
|
|
[279822]
|
|
|
559640
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f7beea9a-81b9-4505-a6b1-ac998b821b60
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f7beea9a-81b9-4505-a6b1-ac998b821b60
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:34
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279820.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279820.0 (TID 279820) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-eac9b7ad-7386-4fdc-abbb-a24c107d477c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279820.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279820.0 (TID 279820) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-eac9b7ad-7386-4fdc-abbb-a24c107d477c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559641
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f7beea9a-81b9-4505-a6b1-ac998b821b60
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f7beea9a-81b9-4505-a6b1-ac998b821b60
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:34
|
11 ms
|
|
[279820]
|
|
|
559410
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f76e5321-7ba6-4f24-a6c2-630f82fbeb0c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f76e5321-7ba6-4f24-a6c2-630f82fbeb0c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:28
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279705.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279705.0 (TID 279705) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8ab96a5f-ef46-4edf-b66a-0bb1eb121091-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279705.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279705.0 (TID 279705) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8ab96a5f-ef46-4edf-b66a-0bb1eb121091-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559411
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f76e5321-7ba6-4f24-a6c2-630f82fbeb0c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f76e5321-7ba6-4f24-a6c2-630f82fbeb0c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:28
|
12 ms
|
|
[279705]
|
|
|
559994
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f701e6f7-97a3-4bd4-a90b-b7be099e6aa8
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f701e6f7-97a3-4bd4-a90b-b7be099e6aa8
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:41
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279997.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279997.0 (TID 279997) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ef092cfd-299f-402e-aae8-67a4536592d7-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279997.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279997.0 (TID 279997) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ef092cfd-299f-402e-aae8-67a4536592d7-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559995
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f701e6f7-97a3-4bd4-a90b-b7be099e6aa8
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f701e6f7-97a3-4bd4-a90b-b7be099e6aa8
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:41
|
11 ms
|
|
[279997]
|
|
|
559222
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f6b1bb6c-8273-4355-82c7-347e6df9e7b0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f6b1bb6c-8273-4355-82c7-347e6df9e7b0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:24
|
19 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279611.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279611.0 (TID 279611) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8cffe480-f1db-410c-86ed-53d145cca851-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279611.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279611.0 (TID 279611) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8cffe480-f1db-410c-86ed-53d145cca851-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559223
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f6b1bb6c-8273-4355-82c7-347e6df9e7b0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f6b1bb6c-8273-4355-82c7-347e6df9e7b0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:24
|
16 ms
|
|
[279611]
|
|
|
559974
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f4c845a6-598b-42ec-9656-6cdc0c48cee9
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f4c845a6-598b-42ec-9656-6cdc0c48cee9
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:41
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279987.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279987.0 (TID 279987) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f632b19c-7314-4618-8f7f-e286f1b93fb7-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279987.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279987.0 (TID 279987) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f632b19c-7314-4618-8f7f-e286f1b93fb7-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559975
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f4c845a6-598b-42ec-9656-6cdc0c48cee9
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f4c845a6-598b-42ec-9656-6cdc0c48cee9
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:41
|
14 ms
|
|
[279987]
|
|
|
560056
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f2f3e57d-ab88-4426-860d-29fccdeef30f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f2f3e57d-ab88-4426-860d-29fccdeef30f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:42
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 280028.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280028.0 (TID 280028) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-82563d34-9a16-4389-81af-6e6f3c17a00e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 280028.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280028.0 (TID 280028) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-82563d34-9a16-4389-81af-6e6f3c17a00e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
560057
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f2f3e57d-ab88-4426-860d-29fccdeef30f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f2f3e57d-ab88-4426-860d-29fccdeef30f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:42
|
14 ms
|
|
[280028]
|
|
|
559746
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f16170d3-7cf4-4ad1-abb6-fe855c734050
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f16170d3-7cf4-4ad1-abb6-fe855c734050
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:36
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279873.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279873.0 (TID 279873) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ce9b7574-6f64-4e7c-b267-c8c18c5cd57f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279873.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279873.0 (TID 279873) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ce9b7574-6f64-4e7c-b267-c8c18c5cd57f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559747
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f16170d3-7cf4-4ad1-abb6-fe855c734050
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f16170d3-7cf4-4ad1-abb6-fe855c734050
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:36
|
12 ms
|
|
[279873]
|
|
|
560028
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f0f85809-bac4-4ec5-b95a-a0f1e4984536
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f0f85809-bac4-4ec5-b95a-a0f1e4984536
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:42
|
23 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 280014.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280014.0 (TID 280014) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3fa6b071-c468-4420-a622-74a53a7c6113-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 280014.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280014.0 (TID 280014) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3fa6b071-c468-4420-a622-74a53a7c6113-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
560029
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f0f85809-bac4-4ec5-b95a-a0f1e4984536
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f0f85809-bac4-4ec5-b95a-a0f1e4984536
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:42
|
20 ms
|
|
[280014]
|
|
|
559320
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f0704ed4-9ddb-4e66-b1a3-11d93d59d1ae
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f0704ed4-9ddb-4e66-b1a3-11d93d59d1ae
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:26
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279660.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279660.0 (TID 279660) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-0a13e532-fe7c-47a7-ba29-72c4ea4ea169-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279660.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279660.0 (TID 279660) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-0a13e532-fe7c-47a7-ba29-72c4ea4ea169-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559321
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f0704ed4-9ddb-4e66-b1a3-11d93d59d1ae
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f0704ed4-9ddb-4e66-b1a3-11d93d59d1ae
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:26
|
12 ms
|
|
[279660]
|
|
|
559352
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f0370987-47cd-4279-9d0e-c9fc83eaeb85
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f0370987-47cd-4279-9d0e-c9fc83eaeb85
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:27
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279676.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279676.0 (TID 279676) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-40b69e7d-b017-4bbf-8a05-1e2650d54eba-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279676.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279676.0 (TID 279676) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-40b69e7d-b017-4bbf-8a05-1e2650d54eba-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559353
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f0370987-47cd-4279-9d0e-c9fc83eaeb85
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f0370987-47cd-4279-9d0e-c9fc83eaeb85
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:27
|
13 ms
|
|
[279676]
|
|
|
559404
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f02463f3-5412-49da-bc44-c32595e7048b
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f02463f3-5412-49da-bc44-c32595e7048b
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:28
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279702.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279702.0 (TID 279702) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-92989737-cad9-45f1-b891-fb666ebdb617-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279702.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279702.0 (TID 279702) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-92989737-cad9-45f1-b891-fb666ebdb617-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559405
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f02463f3-5412-49da-bc44-c32595e7048b
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f02463f3-5412-49da-bc44-c32595e7048b
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:28
|
13 ms
|
|
[279702]
|
|
|
560020
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = efd2e0c7-9aed-441d-b626-6da6bf50eb11
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = efd2e0c7-9aed-441d-b626-6da6bf50eb11
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:42
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 280010.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280010.0 (TID 280010) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c6cb2042-7da8-476c-beca-fd44282b86e9-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 280010.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280010.0 (TID 280010) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c6cb2042-7da8-476c-beca-fd44282b86e9-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
560021
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = efd2e0c7-9aed-441d-b626-6da6bf50eb11
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = efd2e0c7-9aed-441d-b626-6da6bf50eb11
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:42
|
11 ms
|
|
[280010]
|
|
|
560104
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ee06152a-da6f-4445-8e34-4629ccbf3373
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ee06152a-da6f-4445-8e34-4629ccbf3373
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:44
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 280052.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280052.0 (TID 280052) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c9b6e793-b8d6-4b41-a183-751801273daa-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 280052.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280052.0 (TID 280052) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c9b6e793-b8d6-4b41-a183-751801273daa-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
560105
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ee06152a-da6f-4445-8e34-4629ccbf3373
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ee06152a-da6f-4445-8e34-4629ccbf3373
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:44
|
11 ms
|
|
[280052]
|
|
|
559952
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ed656470-9c2f-44e7-9dc6-202566dfffd7
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ed656470-9c2f-44e7-9dc6-202566dfffd7
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:40
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279976.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279976.0 (TID 279976) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-bd6c3089-77ba-4733-8535-4baf6aa1fea2-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279976.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279976.0 (TID 279976) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-bd6c3089-77ba-4733-8535-4baf6aa1fea2-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559953
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ed656470-9c2f-44e7-9dc6-202566dfffd7
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ed656470-9c2f-44e7-9dc6-202566dfffd7
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:40
|
11 ms
|
|
[279976]
|
|
|
559724
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ec5f6e67-3fbd-422e-a3c5-0d7481b9aad8
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ec5f6e67-3fbd-422e-a3c5-0d7481b9aad8
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:35
|
22 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279862.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279862.0 (TID 279862) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2a9d8a54-c3fb-45f8-a845-f0235b8f1d97-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279862.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279862.0 (TID 279862) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2a9d8a54-c3fb-45f8-a845-f0235b8f1d97-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559725
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ec5f6e67-3fbd-422e-a3c5-0d7481b9aad8
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ec5f6e67-3fbd-422e-a3c5-0d7481b9aad8
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:35
|
12 ms
|
|
[279862]
|
|
|
559288
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ec04b9a9-0c9f-453c-b9d7-b546b2c7f981
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ec04b9a9-0c9f-453c-b9d7-b546b2c7f981
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:26
|
25 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279644.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279644.0 (TID 279644) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ecec9603-4d45-42de-91a3-a79b80b8f313-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279644.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279644.0 (TID 279644) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ecec9603-4d45-42de-91a3-a79b80b8f313-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559289
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ec04b9a9-0c9f-453c-b9d7-b546b2c7f981
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ec04b9a9-0c9f-453c-b9d7-b546b2c7f981
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:26
|
23 ms
|
|
[279644]
|
|
|
559614
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = eaafa27b-b894-4454-82e5-7263c67951ad
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = eaafa27b-b894-4454-82e5-7263c67951ad
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:33
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279807.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279807.0 (TID 279807) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-263eebd7-62d3-41c8-8fcd-8e656dcf837e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279807.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279807.0 (TID 279807) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-263eebd7-62d3-41c8-8fcd-8e656dcf837e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559615
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = eaafa27b-b894-4454-82e5-7263c67951ad
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = eaafa27b-b894-4454-82e5-7263c67951ad
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:33
|
13 ms
|
|
[279807]
|
|
|
560008
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = eaa4c33a-4bf9-422a-a3b8-b7228176c8bc
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = eaa4c33a-4bf9-422a-a3b8-b7228176c8bc
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:41
|
26 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 280004.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280004.0 (TID 280004) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-535a15ff-d993-48e6-b639-16fee3dce241-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 280004.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280004.0 (TID 280004) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-535a15ff-d993-48e6-b639-16fee3dce241-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
560009
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = eaa4c33a-4bf9-422a-a3b8-b7228176c8bc
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = eaa4c33a-4bf9-422a-a3b8-b7228176c8bc
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:41
|
23 ms
|
|
[280004]
|
|
|
559740
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e936ec51-4b15-49a2-9a84-46492d5413b7
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e936ec51-4b15-49a2-9a84-46492d5413b7
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:36
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279870.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279870.0 (TID 279870) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-abcbc9bf-6ac6-4300-a987-cca3f94ae033-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279870.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279870.0 (TID 279870) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-abcbc9bf-6ac6-4300-a987-cca3f94ae033-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559741
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e936ec51-4b15-49a2-9a84-46492d5413b7
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e936ec51-4b15-49a2-9a84-46492d5413b7
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:36
|
11 ms
|
|
[279870]
|
|
|
559836
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e92868ff-0e6e-48be-a254-e8571d0c8d5c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e92868ff-0e6e-48be-a254-e8571d0c8d5c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:38
|
23 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279918.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279918.0 (TID 279918) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d9b6430e-7bee-4fa5-91e0-b55629826da0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279918.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279918.0 (TID 279918) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d9b6430e-7bee-4fa5-91e0-b55629826da0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559837
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e92868ff-0e6e-48be-a254-e8571d0c8d5c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e92868ff-0e6e-48be-a254-e8571d0c8d5c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:38
|
11 ms
|
|
[279918]
|
|
|
559370
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e926a9fe-44cb-4c3c-8dd1-921193fdfe6d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e926a9fe-44cb-4c3c-8dd1-921193fdfe6d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:28
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279685.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279685.0 (TID 279685) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a33b71e9-d0fe-49c0-be77-9d65e32b1ec8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279685.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279685.0 (TID 279685) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a33b71e9-d0fe-49c0-be77-9d65e32b1ec8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559371
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e926a9fe-44cb-4c3c-8dd1-921193fdfe6d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e926a9fe-44cb-4c3c-8dd1-921193fdfe6d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:28
|
11 ms
|
|
[279685]
|
|
|
559324
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e8271b5d-32d4-459b-9666-215f58225baf
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e8271b5d-32d4-459b-9666-215f58225baf
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:27
|
19 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279662.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279662.0 (TID 279662) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1c337728-f79a-4c77-a297-003896985b03-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279662.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279662.0 (TID 279662) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1c337728-f79a-4c77-a297-003896985b03-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559325
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e8271b5d-32d4-459b-9666-215f58225baf
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e8271b5d-32d4-459b-9666-215f58225baf
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:27
|
15 ms
|
|
[279662]
|
|
|
560106
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e7f47e5c-156d-4500-b742-a89e55878fb4
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e7f47e5c-156d-4500-b742-a89e55878fb4
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:44
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 280053.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280053.0 (TID 280053) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-dc5f3d1d-f409-4442-b00c-7efb93a5e641-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 280053.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280053.0 (TID 280053) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-dc5f3d1d-f409-4442-b00c-7efb93a5e641-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
560107
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e7f47e5c-156d-4500-b742-a89e55878fb4
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e7f47e5c-156d-4500-b742-a89e55878fb4
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:44
|
12 ms
|
|
[280053]
|
|
|
559858
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e7707252-4c1c-466d-b517-574edfd8ded8
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e7707252-4c1c-466d-b517-574edfd8ded8
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:38
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279929.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279929.0 (TID 279929) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6934f750-976f-487d-8948-14f7cc6df4de-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279929.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279929.0 (TID 279929) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6934f750-976f-487d-8948-14f7cc6df4de-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559859
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e7707252-4c1c-466d-b517-574edfd8ded8
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e7707252-4c1c-466d-b517-574edfd8ded8
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:38
|
11 ms
|
|
[279929]
|
|
|
559716
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e75084bb-05c6-47aa-b36a-ccb2db17d800
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e75084bb-05c6-47aa-b36a-ccb2db17d800
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:35
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279858.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279858.0 (TID 279858) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f4330d39-ee7f-49e2-ba6b-e129e89b55a7-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279858.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279858.0 (TID 279858) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f4330d39-ee7f-49e2-ba6b-e129e89b55a7-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559717
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e75084bb-05c6-47aa-b36a-ccb2db17d800
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e75084bb-05c6-47aa-b36a-ccb2db17d800
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:35
|
13 ms
|
|
[279858]
|
|
|
559802
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e71a065b-6f63-429b-b7fd-e4ceec5e68af
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e71a065b-6f63-429b-b7fd-e4ceec5e68af
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:37
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279901.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279901.0 (TID 279901) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-04e1b91d-79c7-417f-aa57-78c14ce2a914-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279901.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279901.0 (TID 279901) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-04e1b91d-79c7-417f-aa57-78c14ce2a914-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559803
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e71a065b-6f63-429b-b7fd-e4ceec5e68af
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e71a065b-6f63-429b-b7fd-e4ceec5e68af
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:37
|
14 ms
|
|
[279901]
|
|
|
559178
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e674edd2-3c85-4e02-9aff-fa9703551934
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e674edd2-3c85-4e02-9aff-fa9703551934
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:23
|
18 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279589.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279589.0 (TID 279589) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-01157a96-9af2-4cf8-9c08-78036808424e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279589.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279589.0 (TID 279589) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-01157a96-9af2-4cf8-9c08-78036808424e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559179
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e674edd2-3c85-4e02-9aff-fa9703551934
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e674edd2-3c85-4e02-9aff-fa9703551934
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:23
|
15 ms
|
|
[279589]
|
|
|
559578
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e614485c-49a5-4a32-bf32-140250fb21bd
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e614485c-49a5-4a32-bf32-140250fb21bd
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:32
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279789.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279789.0 (TID 279789) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-99639f3d-6f40-47b9-974d-f2348d16c212-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279789.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279789.0 (TID 279789) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-99639f3d-6f40-47b9-974d-f2348d16c212-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559579
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e614485c-49a5-4a32-bf32-140250fb21bd
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e614485c-49a5-4a32-bf32-140250fb21bd
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:32
|
12 ms
|
|
[279789]
|
|
|
559304
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e50b35dd-9029-4aba-aa59-235cde0e77f5
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e50b35dd-9029-4aba-aa59-235cde0e77f5
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:26
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279652.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279652.0 (TID 279652) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ee400618-8d63-424a-9060-15bdb5df3538-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279652.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279652.0 (TID 279652) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ee400618-8d63-424a-9060-15bdb5df3538-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559305
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e50b35dd-9029-4aba-aa59-235cde0e77f5
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e50b35dd-9029-4aba-aa59-235cde0e77f5
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:26
|
10 ms
|
|
[279652]
|
|
|
559332
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e482444b-9564-46c3-abe3-74150abd1ccf
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e482444b-9564-46c3-abe3-74150abd1ccf
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:27
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279666.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279666.0 (TID 279666) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-151d5a8e-1b4b-4445-928c-2f7fb406b630-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279666.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279666.0 (TID 279666) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-151d5a8e-1b4b-4445-928c-2f7fb406b630-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559333
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e482444b-9564-46c3-abe3-74150abd1ccf
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e482444b-9564-46c3-abe3-74150abd1ccf
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:27
|
12 ms
|
|
[279666]
|
|
|
559862
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e464dc29-61c2-444c-9ce6-141ad9f0ce59
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e464dc29-61c2-444c-9ce6-141ad9f0ce59
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:38
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279931.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279931.0 (TID 279931) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-7b27dcb6-3ccc-4159-a2d5-094f97d308be-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279931.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279931.0 (TID 279931) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-7b27dcb6-3ccc-4159-a2d5-094f97d308be-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559863
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e464dc29-61c2-444c-9ce6-141ad9f0ce59
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e464dc29-61c2-444c-9ce6-141ad9f0ce59
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:38
|
12 ms
|
|
[279931]
|
|
|
559608
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e44c8ecb-ac26-41ed-b6a6-33f96a7e95e5
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e44c8ecb-ac26-41ed-b6a6-33f96a7e95e5
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:33
|
37 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279804.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279804.0 (TID 279804) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9cc6f4b7-d39d-4db6-9aa2-8cc2ff8f292d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279804.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279804.0 (TID 279804) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9cc6f4b7-d39d-4db6-9aa2-8cc2ff8f292d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559609
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e44c8ecb-ac26-41ed-b6a6-33f96a7e95e5
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e44c8ecb-ac26-41ed-b6a6-33f96a7e95e5
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:33
|
33 ms
|
|
[279804]
|
|
|
559472
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e3c4c2eb-f552-4029-b122-e1e9ffa11930
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e3c4c2eb-f552-4029-b122-e1e9ffa11930
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:30
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279736.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279736.0 (TID 279736) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ac10560e-ea2a-41b6-ba18-981be6e66f5d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279736.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279736.0 (TID 279736) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ac10560e-ea2a-41b6-ba18-981be6e66f5d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559473
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e3c4c2eb-f552-4029-b122-e1e9ffa11930
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e3c4c2eb-f552-4029-b122-e1e9ffa11930
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:30
|
10 ms
|
|
[279736]
|
|
|
559426
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e3bdb65f-e6e0-4648-ac24-4edc4f8ce45c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e3bdb65f-e6e0-4648-ac24-4edc4f8ce45c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:29
|
18 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279713.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279713.0 (TID 279713) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f00c9181-2597-4187-80d8-0b04f3bb76f6-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279713.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279713.0 (TID 279713) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f00c9181-2597-4187-80d8-0b04f3bb76f6-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559427
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e3bdb65f-e6e0-4648-ac24-4edc4f8ce45c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e3bdb65f-e6e0-4648-ac24-4edc4f8ce45c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:29
|
15 ms
|
|
[279713]
|
|
|
559340
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e26dbc2f-1dcc-470e-99e5-9e5a5a26502c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e26dbc2f-1dcc-470e-99e5-9e5a5a26502c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:27
|
18 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279670.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279670.0 (TID 279670) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d81a30ef-10b5-4c83-ab31-6e03a076d187-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279670.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279670.0 (TID 279670) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d81a30ef-10b5-4c83-ab31-6e03a076d187-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559341
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e26dbc2f-1dcc-470e-99e5-9e5a5a26502c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e26dbc2f-1dcc-470e-99e5-9e5a5a26502c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:27
|
14 ms
|
|
[279670]
|
|
|
559586
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e2696c8d-6a4f-4b91-892f-436014a17c4b
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e2696c8d-6a4f-4b91-892f-436014a17c4b
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:32
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279793.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279793.0 (TID 279793) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-069d764d-0785-4bbc-878f-3b389cb93e5c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279793.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279793.0 (TID 279793) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-069d764d-0785-4bbc-878f-3b389cb93e5c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559587
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e2696c8d-6a4f-4b91-892f-436014a17c4b
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e2696c8d-6a4f-4b91-892f-436014a17c4b
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:32
|
14 ms
|
|
[279793]
|
|
|
559348
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1fed127-863f-4a46-b188-422ae0e3f432
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1fed127-863f-4a46-b188-422ae0e3f432
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:27
|
28 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279674.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279674.0 (TID 279674) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-fdfe1240-bb86-4c06-9c96-2f009664d65c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279674.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279674.0 (TID 279674) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-fdfe1240-bb86-4c06-9c96-2f009664d65c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559349
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1fed127-863f-4a46-b188-422ae0e3f432
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1fed127-863f-4a46-b188-422ae0e3f432
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:27
|
25 ms
|
|
[279674]
|
|
|
559174
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1feab89-c8b5-4467-886d-64a43eeacea2
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1feab89-c8b5-4467-886d-64a43eeacea2
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:23
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279587.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279587.0 (TID 279587) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-0e1f4dd3-768b-4c21-8917-948b7cd61745-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279587.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279587.0 (TID 279587) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-0e1f4dd3-768b-4c21-8917-948b7cd61745-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559175
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1feab89-c8b5-4467-886d-64a43eeacea2
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1feab89-c8b5-4467-886d-64a43eeacea2
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:23
|
12 ms
|
|
[279587]
|
|
|
559846
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1a3e3fb-9717-499c-835b-1d4dc4af60f3
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1a3e3fb-9717-499c-835b-1d4dc4af60f3
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:38
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279923.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279923.0 (TID 279923) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-fcf1246c-d309-47f5-b024-59ffe01e0646-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279923.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279923.0 (TID 279923) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-fcf1246c-d309-47f5-b024-59ffe01e0646-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559847
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1a3e3fb-9717-499c-835b-1d4dc4af60f3
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1a3e3fb-9717-499c-835b-1d4dc4af60f3
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:38
|
11 ms
|
|
[279923]
|
|
|
559456
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e10787dd-2917-4a22-83c8-acab9bfc98c0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e10787dd-2917-4a22-83c8-acab9bfc98c0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:30
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279728.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279728.0 (TID 279728) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-7b2f5271-26ef-4b67-ad01-e24d33dd1278-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279728.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279728.0 (TID 279728) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-7b2f5271-26ef-4b67-ad01-e24d33dd1278-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559457
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e10787dd-2917-4a22-83c8-acab9bfc98c0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e10787dd-2917-4a22-83c8-acab9bfc98c0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:30
|
11 ms
|
|
[279728]
|
|
|
559486
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e08385e4-52e6-48dd-82bb-df6e80c112b5
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e08385e4-52e6-48dd-82bb-df6e80c112b5
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:30
|
12 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279743.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279743.0 (TID 279743) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-80279614-7d9f-43af-ba04-b853301431ea-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279743.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279743.0 (TID 279743) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-80279614-7d9f-43af-ba04-b853301431ea-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559487
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e08385e4-52e6-48dd-82bb-df6e80c112b5
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e08385e4-52e6-48dd-82bb-df6e80c112b5
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:30
|
10 ms
|
|
[279743]
|
|
|
559542
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e03fb8ac-7d62-444f-881f-2ba43f6abd3a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e03fb8ac-7d62-444f-881f-2ba43f6abd3a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:31
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279771.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279771.0 (TID 279771) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ac210679-fc81-4758-960d-d2b432956290-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279771.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279771.0 (TID 279771) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ac210679-fc81-4758-960d-d2b432956290-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559543
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e03fb8ac-7d62-444f-881f-2ba43f6abd3a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e03fb8ac-7d62-444f-881f-2ba43f6abd3a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:31
|
12 ms
|
|
[279771]
|
|
|
559940
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e0270a8a-27cf-4e1a-956b-012321af0c13
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e0270a8a-27cf-4e1a-956b-012321af0c13
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:40
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279970.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279970.0 (TID 279970) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-32f55174-b5b6-461a-841a-ba2413093c20-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279970.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279970.0 (TID 279970) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-32f55174-b5b6-461a-841a-ba2413093c20-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559941
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e0270a8a-27cf-4e1a-956b-012321af0c13
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e0270a8a-27cf-4e1a-956b-012321af0c13
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:40
|
12 ms
|
|
[279970]
|
|
|
559316
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = df75215b-cdda-430c-9073-52483be25d03
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = df75215b-cdda-430c-9073-52483be25d03
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:26
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279658.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279658.0 (TID 279658) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-7a80e93e-3ec8-4aa8-ba6d-4c831d3282a3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279658.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279658.0 (TID 279658) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-7a80e93e-3ec8-4aa8-ba6d-4c831d3282a3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559317
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = df75215b-cdda-430c-9073-52483be25d03
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = df75215b-cdda-430c-9073-52483be25d03
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:26
|
13 ms
|
|
[279658]
|
|
|
559680
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = df355681-55c9-426d-8f2a-ad72e5fdb509
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = df355681-55c9-426d-8f2a-ad72e5fdb509
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:34
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279840.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279840.0 (TID 279840) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ca50a338-f38c-443a-9ef8-05ac74edcb8b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279840.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279840.0 (TID 279840) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ca50a338-f38c-443a-9ef8-05ac74edcb8b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559681
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = df355681-55c9-426d-8f2a-ad72e5fdb509
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = df355681-55c9-426d-8f2a-ad72e5fdb509
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:34
|
12 ms
|
|
[279840]
|
|
|
559622
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = deb627f3-7396-4de5-8861-6b4923835ee8
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = deb627f3-7396-4de5-8861-6b4923835ee8
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:33
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279811.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279811.0 (TID 279811) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-58169ac5-af84-4968-ad4f-42bcfe5c8b60-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279811.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279811.0 (TID 279811) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-58169ac5-af84-4968-ad4f-42bcfe5c8b60-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559623
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = deb627f3-7396-4de5-8861-6b4923835ee8
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = deb627f3-7396-4de5-8861-6b4923835ee8
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:33
|
13 ms
|
|
[279811]
|
|
|
559812
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dcdd01c3-47b7-4f2a-937b-1d161e818b77
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dcdd01c3-47b7-4f2a-937b-1d161e818b77
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:37
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279906.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279906.0 (TID 279906) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8594e764-6488-4bf4-bd42-2ec2f67696f8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279906.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279906.0 (TID 279906) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8594e764-6488-4bf4-bd42-2ec2f67696f8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559813
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dcdd01c3-47b7-4f2a-937b-1d161e818b77
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dcdd01c3-47b7-4f2a-937b-1d161e818b77
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:37
|
11 ms
|
|
[279906]
|
|
|
560018
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dc779965-bb51-49b6-b5b2-fabd650d0560
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dc779965-bb51-49b6-b5b2-fabd650d0560
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:42
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 280009.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280009.0 (TID 280009) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-af002c3d-fac4-45b7-9e27-05aa1b3299db-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 280009.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280009.0 (TID 280009) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-af002c3d-fac4-45b7-9e27-05aa1b3299db-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
560019
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dc779965-bb51-49b6-b5b2-fabd650d0560
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dc779965-bb51-49b6-b5b2-fabd650d0560
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:42
|
12 ms
|
|
[280009]
|
|
|
559570
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dbc27609-b031-4a49-b805-c2290db6f6e1
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dbc27609-b031-4a49-b805-c2290db6f6e1
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:32
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279785.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279785.0 (TID 279785) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2ceeed12-91c1-4cef-b0cb-6d4de7ebb197-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279785.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279785.0 (TID 279785) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2ceeed12-91c1-4cef-b0cb-6d4de7ebb197-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559571
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dbc27609-b031-4a49-b805-c2290db6f6e1
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dbc27609-b031-4a49-b805-c2290db6f6e1
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:32
|
11 ms
|
|
[279785]
|
|
|
559732
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = daf5a5dc-494d-4f1d-9a95-3688773f6228
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = daf5a5dc-494d-4f1d-9a95-3688773f6228
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:35
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279866.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279866.0 (TID 279866) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d4421cae-4dcf-4bf6-b926-7eaa2a8b0e1d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279866.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279866.0 (TID 279866) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d4421cae-4dcf-4bf6-b926-7eaa2a8b0e1d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559733
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = daf5a5dc-494d-4f1d-9a95-3688773f6228
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = daf5a5dc-494d-4f1d-9a95-3688773f6228
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:35
|
12 ms
|
|
[279866]
|
|
|
559686
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dac018ee-2c28-451a-8b62-32bff67b3cfa
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dac018ee-2c28-451a-8b62-32bff67b3cfa
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:34
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279843.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279843.0 (TID 279843) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-47b0c472-f0ff-49aa-aef3-8b0848ee48a2-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279843.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279843.0 (TID 279843) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-47b0c472-f0ff-49aa-aef3-8b0848ee48a2-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559687
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dac018ee-2c28-451a-8b62-32bff67b3cfa
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dac018ee-2c28-451a-8b62-32bff67b3cfa
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:34
|
12 ms
|
|
[279843]
|
|
|
559254
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d9da453a-f2f0-48a1-bde0-8d0975ef17b4
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d9da453a-f2f0-48a1-bde0-8d0975ef17b4
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:25
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279627.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279627.0 (TID 279627) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-89ffe82d-de44-4813-9478-e43d560a3acf-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279627.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279627.0 (TID 279627) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-89ffe82d-de44-4813-9478-e43d560a3acf-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559255
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d9da453a-f2f0-48a1-bde0-8d0975ef17b4
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d9da453a-f2f0-48a1-bde0-8d0975ef17b4
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:25
|
11 ms
|
|
[279627]
|
|
|
559134
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d9cf2d57-a4d9-4d65-b013-316149667d96
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d9cf2d57-a4d9-4d65-b013-316149667d96
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:22
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279567.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279567.0 (TID 279567) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d5f3b68b-4e65-4b0a-986e-9a22af27de29-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279567.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279567.0 (TID 279567) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d5f3b68b-4e65-4b0a-986e-9a22af27de29-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559135
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d9cf2d57-a4d9-4d65-b013-316149667d96
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d9cf2d57-a4d9-4d65-b013-316149667d96
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:22
|
10 ms
|
|
[279567]
|
|
|
559138
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d91e2328-0a1f-43ed-9961-0cc52ac15d44
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d91e2328-0a1f-43ed-9961-0cc52ac15d44
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:23
|
19 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279569.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279569.0 (TID 279569) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3f56ede8-49ea-4464-93b4-c263417fe57c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279569.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279569.0 (TID 279569) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3f56ede8-49ea-4464-93b4-c263417fe57c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559139
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d91e2328-0a1f-43ed-9961-0cc52ac15d44
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d91e2328-0a1f-43ed-9961-0cc52ac15d44
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:23
|
16 ms
|
|
[279569]
|
|
|
559440
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d87e5605-db82-4f53-ab28-d2e7fdb6bdbc
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d87e5605-db82-4f53-ab28-d2e7fdb6bdbc
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:29
|
18 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279720.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279720.0 (TID 279720) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b3c81b8f-a2db-419d-878f-0c646f36922a-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279720.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279720.0 (TID 279720) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b3c81b8f-a2db-419d-878f-0c646f36922a-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559441
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d87e5605-db82-4f53-ab28-d2e7fdb6bdbc
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d87e5605-db82-4f53-ab28-d2e7fdb6bdbc
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:29
|
14 ms
|
|
[279720]
|
|
|
559898
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d875e5b1-0d28-40b4-b239-581f5fa24184
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d875e5b1-0d28-40b4-b239-581f5fa24184
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:39
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279949.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279949.0 (TID 279949) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-29608b2c-6ddc-4246-8580-e21e08f70dc1-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279949.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279949.0 (TID 279949) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-29608b2c-6ddc-4246-8580-e21e08f70dc1-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559899
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d875e5b1-0d28-40b4-b239-581f5fa24184
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d875e5b1-0d28-40b4-b239-581f5fa24184
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:39
|
13 ms
|
|
[279949]
|
|
|
559658
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d83e09f7-fc71-4c49-94a2-56bcdbed5cfd
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d83e09f7-fc71-4c49-94a2-56bcdbed5cfd
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:34
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279829.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279829.0 (TID 279829) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-99d5c203-142e-4625-88be-088f240307e9-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279829.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279829.0 (TID 279829) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-99d5c203-142e-4625-88be-088f240307e9-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559659
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d83e09f7-fc71-4c49-94a2-56bcdbed5cfd
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d83e09f7-fc71-4c49-94a2-56bcdbed5cfd
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:34
|
10 ms
|
|
[279829]
|
|
|
559198
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d8146737-03bc-43c1-8ab5-9078924ba725
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d8146737-03bc-43c1-8ab5-9078924ba725
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:24
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279599.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279599.0 (TID 279599) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-51bb8d9f-823d-442c-bd31-5f7c356bcb13-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279599.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279599.0 (TID 279599) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-51bb8d9f-823d-442c-bd31-5f7c356bcb13-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559199
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d8146737-03bc-43c1-8ab5-9078924ba725
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d8146737-03bc-43c1-8ab5-9078924ba725
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:24
|
11 ms
|
|
[279599]
|
|
|
559408
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d7f1c23e-2ebb-4f56-a501-e1fbe2e1edfe
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d7f1c23e-2ebb-4f56-a501-e1fbe2e1edfe
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:28
|
26 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279704.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279704.0 (TID 279704) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5fba45b4-9077-4c24-9de9-de54ac736e01-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279704.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279704.0 (TID 279704) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5fba45b4-9077-4c24-9de9-de54ac736e01-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559409
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d7f1c23e-2ebb-4f56-a501-e1fbe2e1edfe
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d7f1c23e-2ebb-4f56-a501-e1fbe2e1edfe
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:28
|
24 ms
|
|
[279704]
|
|
|
559938
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d765efb2-7e6f-4552-9a8d-eff027fefbcb
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d765efb2-7e6f-4552-9a8d-eff027fefbcb
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:40
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279969.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279969.0 (TID 279969) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-34155d33-ad88-4595-8912-3d888c0184ab-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279969.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279969.0 (TID 279969) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-34155d33-ad88-4595-8912-3d888c0184ab-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559939
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d765efb2-7e6f-4552-9a8d-eff027fefbcb
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d765efb2-7e6f-4552-9a8d-eff027fefbcb
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:40
|
12 ms
|
|
[279969]
|
|
|
559752
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d72c28c1-3645-46a0-8528-720a6aaeb03c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d72c28c1-3645-46a0-8528-720a6aaeb03c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:36
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279876.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279876.0 (TID 279876) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-06ef4cc3-11fe-436f-9cd7-de1edc662fbf-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279876.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279876.0 (TID 279876) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-06ef4cc3-11fe-436f-9cd7-de1edc662fbf-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559753
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d72c28c1-3645-46a0-8528-720a6aaeb03c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d72c28c1-3645-46a0-8528-720a6aaeb03c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:36
|
11 ms
|
|
[279876]
|
|
|
559162
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d7092c31-4725-434f-974f-f4cb61a0c1a7
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d7092c31-4725-434f-974f-f4cb61a0c1a7
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:23
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279581.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279581.0 (TID 279581) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f81fee29-1885-492a-a85d-dfbd79c03611-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279581.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279581.0 (TID 279581) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f81fee29-1885-492a-a85d-dfbd79c03611-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559163
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d7092c31-4725-434f-974f-f4cb61a0c1a7
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d7092c31-4725-434f-974f-f4cb61a0c1a7
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:23
|
12 ms
|
|
[279581]
|
|
|
560094
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d61b5ae6-15f1-4964-be5c-dde87fa33c4f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d61b5ae6-15f1-4964-be5c-dde87fa33c4f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:43
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 280047.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280047.0 (TID 280047) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6bf92859-e961-48d0-96ef-4d61c96ce184-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 280047.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280047.0 (TID 280047) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6bf92859-e961-48d0-96ef-4d61c96ce184-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
560095
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d61b5ae6-15f1-4964-be5c-dde87fa33c4f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d61b5ae6-15f1-4964-be5c-dde87fa33c4f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:43
|
13 ms
|
|
[280047]
|
|
|
559904
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d5058ea1-3540-4ac8-bd6a-db70b925961e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d5058ea1-3540-4ac8-bd6a-db70b925961e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:39
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279952.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279952.0 (TID 279952) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-efcf6a9d-690d-40c0-b82f-8708bb78c3d8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279952.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279952.0 (TID 279952) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-efcf6a9d-690d-40c0-b82f-8708bb78c3d8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559905
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d5058ea1-3540-4ac8-bd6a-db70b925961e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d5058ea1-3540-4ac8-bd6a-db70b925961e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:39
|
13 ms
|
|
[279952]
|
|
|
559962
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4c416d8-117f-419b-bbc1-4427f128420a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4c416d8-117f-419b-bbc1-4427f128420a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:40
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279981.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279981.0 (TID 279981) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9b747c53-ee73-4bd4-a2c3-3fe62fd3046e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279981.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279981.0 (TID 279981) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9b747c53-ee73-4bd4-a2c3-3fe62fd3046e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559963
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4c416d8-117f-419b-bbc1-4427f128420a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4c416d8-117f-419b-bbc1-4427f128420a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:40
|
14 ms
|
|
[279981]
|
|
|
559296
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d49f6c8b-5533-47e5-a8cf-dd6620076d1d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d49f6c8b-5533-47e5-a8cf-dd6620076d1d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:26
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279648.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279648.0 (TID 279648) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-16bc2dbd-bf29-4b17-9534-75fc96076879-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279648.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279648.0 (TID 279648) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-16bc2dbd-bf29-4b17-9534-75fc96076879-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559297
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d49f6c8b-5533-47e5-a8cf-dd6620076d1d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d49f6c8b-5533-47e5-a8cf-dd6620076d1d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:26
|
11 ms
|
|
[279648]
|
|
|
559428
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d479c275-82e6-442c-b6a6-18d9f557562f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d479c275-82e6-442c-b6a6-18d9f557562f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:29
|
40 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279714.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279714.0 (TID 279714) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-89ea6a46-f51a-444c-9aa0-f246d6096d9f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279714.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279714.0 (TID 279714) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-89ea6a46-f51a-444c-9aa0-f246d6096d9f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559429
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d479c275-82e6-442c-b6a6-18d9f557562f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d479c275-82e6-442c-b6a6-18d9f557562f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:29
|
37 ms
|
|
[279714]
|
|
|
559464
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4518b67-8986-426c-a5f5-03c92a02998f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4518b67-8986-426c-a5f5-03c92a02998f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:30
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279732.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279732.0 (TID 279732) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c85dcec7-ebac-4d5e-8636-944a2589bbc7-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279732.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279732.0 (TID 279732) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c85dcec7-ebac-4d5e-8636-944a2589bbc7-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559465
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4518b67-8986-426c-a5f5-03c92a02998f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4518b67-8986-426c-a5f5-03c92a02998f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:30
|
11 ms
|
|
[279732]
|
|
|
559528
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4257c1f-2bcd-4ce8-b79c-e274af47fc1c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4257c1f-2bcd-4ce8-b79c-e274af47fc1c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:31
|
24 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279764.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279764.0 (TID 279764) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d8125eff-a4c9-467c-8a3f-f2c081beff7e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279764.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279764.0 (TID 279764) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d8125eff-a4c9-467c-8a3f-f2c081beff7e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559529
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4257c1f-2bcd-4ce8-b79c-e274af47fc1c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4257c1f-2bcd-4ce8-b79c-e274af47fc1c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:31
|
21 ms
|
|
[279764]
|
|
|
559568
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d3f98865-4cb2-4e68-b21b-54acff354d6f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d3f98865-4cb2-4e68-b21b-54acff354d6f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:32
|
22 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279784.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279784.0 (TID 279784) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-434cdcd9-4992-4b29-8271-76e15cb4b02a-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279784.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279784.0 (TID 279784) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-434cdcd9-4992-4b29-8271-76e15cb4b02a-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559569
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d3f98865-4cb2-4e68-b21b-54acff354d6f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d3f98865-4cb2-4e68-b21b-54acff354d6f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:32
|
20 ms
|
|
[279784]
|
|
|
559792
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d321f4e7-8093-476d-9235-28585b9c1431
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d321f4e7-8093-476d-9235-28585b9c1431
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:37
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279896.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279896.0 (TID 279896) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-904287be-ab08-4e89-9dfd-650f6df7bf90-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279896.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279896.0 (TID 279896) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-904287be-ab08-4e89-9dfd-650f6df7bf90-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559793
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d321f4e7-8093-476d-9235-28585b9c1431
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d321f4e7-8093-476d-9235-28585b9c1431
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:37
|
11 ms
|
|
[279896]
|
|
|
559540
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d2fbe2da-f020-4aa9-9f7a-9a2f0d508ce8
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d2fbe2da-f020-4aa9-9f7a-9a2f0d508ce8
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:31
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279770.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279770.0 (TID 279770) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-43606c29-9301-4cac-9a62-c5a6141ee355-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279770.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279770.0 (TID 279770) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-43606c29-9301-4cac-9a62-c5a6141ee355-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559541
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d2fbe2da-f020-4aa9-9f7a-9a2f0d508ce8
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d2fbe2da-f020-4aa9-9f7a-9a2f0d508ce8
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:31
|
13 ms
|
|
[279770]
|
|
|
559842
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d2d6c178-a542-4e0f-8ab8-ed5a8ef985bf
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d2d6c178-a542-4e0f-8ab8-ed5a8ef985bf
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:38
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279921.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279921.0 (TID 279921) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9ef18a21-0856-4394-9682-d0f44e6092ee-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279921.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279921.0 (TID 279921) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9ef18a21-0856-4394-9682-d0f44e6092ee-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559843
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d2d6c178-a542-4e0f-8ab8-ed5a8ef985bf
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d2d6c178-a542-4e0f-8ab8-ed5a8ef985bf
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:38
|
11 ms
|
|
[279921]
|
|
|
559350
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d251a391-8dac-4321-bf6b-632fce1412c4
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d251a391-8dac-4321-bf6b-632fce1412c4
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:27
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279675.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279675.0 (TID 279675) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-cc391a19-d1b4-4feb-9bc1-9b928be1f8f6-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279675.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279675.0 (TID 279675) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-cc391a19-d1b4-4feb-9bc1-9b928be1f8f6-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559351
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d251a391-8dac-4321-bf6b-632fce1412c4
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d251a391-8dac-4321-bf6b-632fce1412c4
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:27
|
12 ms
|
|
[279675]
|
|
|
559182
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d21a3972-80c7-429e-a802-64699766c5a0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d21a3972-80c7-429e-a802-64699766c5a0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:24
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279591.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279591.0 (TID 279591) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-0a38da30-47a5-4dc8-a6af-48f3acbe7571-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279591.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279591.0 (TID 279591) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-0a38da30-47a5-4dc8-a6af-48f3acbe7571-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559183
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d21a3972-80c7-429e-a802-64699766c5a0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d21a3972-80c7-429e-a802-64699766c5a0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:24
|
12 ms
|
|
[279591]
|
|
|
559444
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d1e4f30a-e9ff-49d3-9551-3fa2d3572407
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d1e4f30a-e9ff-49d3-9551-3fa2d3572407
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:29
|
19 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279722.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279722.0 (TID 279722) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-945c7e5b-014d-4b51-b021-53c7a48e0707-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279722.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279722.0 (TID 279722) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-945c7e5b-014d-4b51-b021-53c7a48e0707-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559445
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d1e4f30a-e9ff-49d3-9551-3fa2d3572407
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d1e4f30a-e9ff-49d3-9551-3fa2d3572407
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:29
|
16 ms
|
|
[279722]
|
|
|
559544
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d17dac34-b64f-4e85-8bcf-86d13d8f8d16
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d17dac34-b64f-4e85-8bcf-86d13d8f8d16
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:31
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279772.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279772.0 (TID 279772) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3cf6036e-0733-4ccd-baf3-dc11e4431c31-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279772.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279772.0 (TID 279772) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3cf6036e-0733-4ccd-baf3-dc11e4431c31-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559545
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d17dac34-b64f-4e85-8bcf-86d13d8f8d16
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d17dac34-b64f-4e85-8bcf-86d13d8f8d16
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:31
|
11 ms
|
|
[279772]
|
|
|
559932
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d15ddfd2-2085-48a5-acc8-8cb4324ca802
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d15ddfd2-2085-48a5-acc8-8cb4324ca802
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:40
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279966.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279966.0 (TID 279966) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-349ceb2d-768b-45d8-b010-553eed419bb5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279966.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279966.0 (TID 279966) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-349ceb2d-768b-45d8-b010-553eed419bb5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559933
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d15ddfd2-2085-48a5-acc8-8cb4324ca802
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d15ddfd2-2085-48a5-acc8-8cb4324ca802
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:40
|
11 ms
|
|
[279966]
|
|
|
559406
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cf8da7a2-0b65-4ff9-8145-73ae973f5f84
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cf8da7a2-0b65-4ff9-8145-73ae973f5f84
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:28
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279703.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279703.0 (TID 279703) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c5a44ee9-e9d5-4122-a6d2-971ac5dce1af-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279703.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279703.0 (TID 279703) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c5a44ee9-e9d5-4122-a6d2-971ac5dce1af-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559407
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cf8da7a2-0b65-4ff9-8145-73ae973f5f84
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cf8da7a2-0b65-4ff9-8145-73ae973f5f84
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:28
|
13 ms
|
|
[279703]
|
|
|
559592
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cece7671-a8e3-44ab-b34c-6f3d25c076a9
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cece7671-a8e3-44ab-b34c-6f3d25c076a9
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:32
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279796.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279796.0 (TID 279796) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-de8a2884-7dfb-4b2e-b630-f11074ba4aa5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279796.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279796.0 (TID 279796) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-de8a2884-7dfb-4b2e-b630-f11074ba4aa5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559593
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cece7671-a8e3-44ab-b34c-6f3d25c076a9
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cece7671-a8e3-44ab-b34c-6f3d25c076a9
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:32
|
14 ms
|
|
[279796]
|
|
|
559212
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ceb72373-bb96-4c47-b9f3-4505425fb198
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ceb72373-bb96-4c47-b9f3-4505425fb198
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:24
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279606.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279606.0 (TID 279606) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-899cde21-7d47-465f-8bb0-25d20335436c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279606.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279606.0 (TID 279606) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-899cde21-7d47-465f-8bb0-25d20335436c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559213
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ceb72373-bb96-4c47-b9f3-4505425fb198
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ceb72373-bb96-4c47-b9f3-4505425fb198
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:24
|
11 ms
|
|
[279606]
|
|
|
560112
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cdf50df4-1693-4846-9b64-483e0832bc20
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cdf50df4-1693-4846-9b64-483e0832bc20
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:44
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 280056.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280056.0 (TID 280056) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b0fec09c-83cb-4de4-8993-200b5f4bbf4d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 280056.0 failed 1 times, most recent failure: Lost task 0.0 in stage 280056.0 (TID 280056) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b0fec09c-83cb-4de4-8993-200b5f4bbf4d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
560113
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cdf50df4-1693-4846-9b64-483e0832bc20
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cdf50df4-1693-4846-9b64-483e0832bc20
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:44
|
12 ms
|
|
[280056]
|
|
|
559916
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cdcf6f46-9b63-4f4b-a1b2-04749730fa79
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cdcf6f46-9b63-4f4b-a1b2-04749730fa79
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:39
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 279958.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279958.0 (TID 279958) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b3df813e-dde6-4008-bca1-ccf54e217d6b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 279958.0 failed 1 times, most recent failure: Lost task 0.0 in stage 279958.0 (TID 279958) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b3df813e-dde6-4008-bca1-ccf54e217d6b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
559917
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cdcf6f46-9b63-4f4b-a1b2-04749730fa79
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cdcf6f46-9b63-4f4b-a1b2-04749730fa79
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:52:39
|
12 ms
|
|
[279958]
|
|