|
424386
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 46756671-49e6-4678-9904-f1ecb5cf3a33
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 46756671-49e6-4678-9904-f1ecb5cf3a33
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:45
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212193.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212193.0 (TID 212193) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b5ae2097-3607-45a8-a13a-95db1f0588aa-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212193.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212193.0 (TID 212193) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b5ae2097-3607-45a8-a13a-95db1f0588aa-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424387
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 46756671-49e6-4678-9904-f1ecb5cf3a33
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 46756671-49e6-4678-9904-f1ecb5cf3a33
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:45
|
11 ms
|
|
[212193]
|
|
|
424396
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 78a3a8eb-30a7-4768-b385-7b659bd421b1
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 78a3a8eb-30a7-4768-b385-7b659bd421b1
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:45
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212198.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212198.0 (TID 212198) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ea6fed8c-7147-43ff-afca-230841fec063-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212198.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212198.0 (TID 212198) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ea6fed8c-7147-43ff-afca-230841fec063-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424397
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 78a3a8eb-30a7-4768-b385-7b659bd421b1
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 78a3a8eb-30a7-4768-b385-7b659bd421b1
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:45
|
10 ms
|
|
[212198]
|
|
|
424442
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 94825df4-5a34-4707-9334-19a3f9706a25
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 94825df4-5a34-4707-9334-19a3f9706a25
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:46
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212221.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212221.0 (TID 212221) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-e33e7e82-4607-49e5-be37-e7176ce12312-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212221.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212221.0 (TID 212221) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-e33e7e82-4607-49e5-be37-e7176ce12312-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424443
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 94825df4-5a34-4707-9334-19a3f9706a25
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 94825df4-5a34-4707-9334-19a3f9706a25
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:46
|
11 ms
|
|
[212221]
|
|
|
424464
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 781ec00c-4864-46e6-9b87-aa821ee46d23
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 781ec00c-4864-46e6-9b87-aa821ee46d23
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212232.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212232.0 (TID 212232) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6e56f9bd-5f2e-4185-8099-5be2ef080b72-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212232.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212232.0 (TID 212232) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6e56f9bd-5f2e-4185-8099-5be2ef080b72-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424465
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 781ec00c-4864-46e6-9b87-aa821ee46d23
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 781ec00c-4864-46e6-9b87-aa821ee46d23
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
11 ms
|
|
[212232]
|
|
|
424474
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 9e1612a5-b7f0-49de-8f53-fe70940a68fe
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 9e1612a5-b7f0-49de-8f53-fe70940a68fe
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212237.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212237.0 (TID 212237) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-10cefb10-5f53-4d90-8c55-e3637d0f04b7-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212237.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212237.0 (TID 212237) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-10cefb10-5f53-4d90-8c55-e3637d0f04b7-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424475
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 9e1612a5-b7f0-49de-8f53-fe70940a68fe
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 9e1612a5-b7f0-49de-8f53-fe70940a68fe
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
11 ms
|
|
[212237]
|
|
|
424480
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 92b41fcd-f44a-4282-a825-09773ed00b8c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 92b41fcd-f44a-4282-a825-09773ed00b8c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212240.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212240.0 (TID 212240) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-4b891a18-d002-44f6-8d86-20d014f92c5a-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212240.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212240.0 (TID 212240) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-4b891a18-d002-44f6-8d86-20d014f92c5a-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424481
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 92b41fcd-f44a-4282-a825-09773ed00b8c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 92b41fcd-f44a-4282-a825-09773ed00b8c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
11 ms
|
|
[212240]
|
|
|
424526
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ea8fbcb2-3e0f-43e5-bfb8-51f779d8108d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ea8fbcb2-3e0f-43e5-bfb8-51f779d8108d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212263.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212263.0 (TID 212263) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1236b022-d8c0-46d7-83f9-c82de381c5e0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212263.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212263.0 (TID 212263) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1236b022-d8c0-46d7-83f9-c82de381c5e0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424527
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ea8fbcb2-3e0f-43e5-bfb8-51f779d8108d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ea8fbcb2-3e0f-43e5-bfb8-51f779d8108d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
11 ms
|
|
[212263]
|
|
|
424534
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 78503f0b-d766-48bf-b805-2c17fe8c9b86
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 78503f0b-d766-48bf-b805-2c17fe8c9b86
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212267.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212267.0 (TID 212267) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-4a157d01-f6a5-4468-98b2-0e717539def9-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212267.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212267.0 (TID 212267) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-4a157d01-f6a5-4468-98b2-0e717539def9-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424535
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 78503f0b-d766-48bf-b805-2c17fe8c9b86
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 78503f0b-d766-48bf-b805-2c17fe8c9b86
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
11 ms
|
|
[212267]
|
|
|
424542
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b56f5d40-c621-4758-856d-5c21b0ebb65e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b56f5d40-c621-4758-856d-5c21b0ebb65e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212271.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212271.0 (TID 212271) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-80beb51a-d156-436f-9ba6-3105b4e030ef-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212271.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212271.0 (TID 212271) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-80beb51a-d156-436f-9ba6-3105b4e030ef-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424543
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b56f5d40-c621-4758-856d-5c21b0ebb65e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b56f5d40-c621-4758-856d-5c21b0ebb65e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212271]
|
|
|
424544
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1bee717e-8e74-44fb-a307-42a8407e61a4
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1bee717e-8e74-44fb-a307-42a8407e61a4
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212272.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212272.0 (TID 212272) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a289c2d3-7ee7-4af8-908d-91276fc6c948-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212272.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212272.0 (TID 212272) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a289c2d3-7ee7-4af8-908d-91276fc6c948-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424545
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1bee717e-8e74-44fb-a307-42a8407e61a4
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1bee717e-8e74-44fb-a307-42a8407e61a4
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212272]
|
|
|
424554
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b017ff61-8fa5-4982-85a5-a9050820e695
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b017ff61-8fa5-4982-85a5-a9050820e695
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212277.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212277.0 (TID 212277) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3385322b-5802-499d-8eeb-b2b4f3aaba60-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212277.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212277.0 (TID 212277) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3385322b-5802-499d-8eeb-b2b4f3aaba60-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424555
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b017ff61-8fa5-4982-85a5-a9050820e695
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b017ff61-8fa5-4982-85a5-a9050820e695
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212277]
|
|
|
424566
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = caee02f1-9de9-49e9-8a9c-4430d9191f7b
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = caee02f1-9de9-49e9-8a9c-4430d9191f7b
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212283.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212283.0 (TID 212283) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d770c263-dc3b-445a-a193-a3d0ad55c01e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212283.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212283.0 (TID 212283) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d770c263-dc3b-445a-a193-a3d0ad55c01e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424567
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = caee02f1-9de9-49e9-8a9c-4430d9191f7b
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = caee02f1-9de9-49e9-8a9c-4430d9191f7b
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212283]
|
|
|
424572
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1f3ab826-d5e4-41ca-91a6-c092f1718974
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1f3ab826-d5e4-41ca-91a6-c092f1718974
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212286.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212286.0 (TID 212286) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1de2b61e-f054-4734-bfde-b377263079b0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212286.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212286.0 (TID 212286) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1de2b61e-f054-4734-bfde-b377263079b0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424573
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1f3ab826-d5e4-41ca-91a6-c092f1718974
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1f3ab826-d5e4-41ca-91a6-c092f1718974
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212286]
|
|
|
424574
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d92087e8-1d93-4d66-b58d-511ca68a8d6c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d92087e8-1d93-4d66-b58d-511ca68a8d6c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212287.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212287.0 (TID 212287) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6b8e085a-a164-473d-8085-febe25d32bdb-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212287.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212287.0 (TID 212287) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6b8e085a-a164-473d-8085-febe25d32bdb-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424575
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d92087e8-1d93-4d66-b58d-511ca68a8d6c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d92087e8-1d93-4d66-b58d-511ca68a8d6c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212287]
|
|
|
424576
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b2efb305-8962-44ad-baa0-67b06c41925a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b2efb305-8962-44ad-baa0-67b06c41925a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212288.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212288.0 (TID 212288) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-53ada27b-d0e1-4900-8466-5ee0841ba10d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212288.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212288.0 (TID 212288) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-53ada27b-d0e1-4900-8466-5ee0841ba10d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424577
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b2efb305-8962-44ad-baa0-67b06c41925a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b2efb305-8962-44ad-baa0-67b06c41925a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212288]
|
|
|
424578
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fa7c3059-1b3c-42f9-82e5-5cd561c4cec5
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fa7c3059-1b3c-42f9-82e5-5cd561c4cec5
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212289.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212289.0 (TID 212289) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-269d9e45-1524-4dcb-a36e-b58025996cd1-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212289.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212289.0 (TID 212289) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-269d9e45-1524-4dcb-a36e-b58025996cd1-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424579
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fa7c3059-1b3c-42f9-82e5-5cd561c4cec5
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fa7c3059-1b3c-42f9-82e5-5cd561c4cec5
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212289]
|
|
|
424590
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 76e4a5be-3114-47e7-8b02-357decaa8a89
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 76e4a5be-3114-47e7-8b02-357decaa8a89
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212295.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212295.0 (TID 212295) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b36fc3c0-f7bc-40f6-9a35-42453d9043bb-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212295.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212295.0 (TID 212295) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b36fc3c0-f7bc-40f6-9a35-42453d9043bb-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424591
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 76e4a5be-3114-47e7-8b02-357decaa8a89
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 76e4a5be-3114-47e7-8b02-357decaa8a89
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212295]
|
|
|
424594
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 4a9667b5-fb29-48b6-bcd1-12317f16da34
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 4a9667b5-fb29-48b6-bcd1-12317f16da34
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:50
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212297.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212297.0 (TID 212297) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b97eaede-b3e7-43f0-8e29-e626fceadaee-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212297.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212297.0 (TID 212297) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b97eaede-b3e7-43f0-8e29-e626fceadaee-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424595
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 4a9667b5-fb29-48b6-bcd1-12317f16da34
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 4a9667b5-fb29-48b6-bcd1-12317f16da34
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:50
|
11 ms
|
|
[212297]
|
|
|
424640
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 51db5ed1-0de3-4db0-8fc8-1706182bb5f4
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 51db5ed1-0de3-4db0-8fc8-1706182bb5f4
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212320.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212320.0 (TID 212320) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-10abb3cd-64b1-45a5-8de9-39ee1b292682-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212320.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212320.0 (TID 212320) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-10abb3cd-64b1-45a5-8de9-39ee1b292682-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424641
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 51db5ed1-0de3-4db0-8fc8-1706182bb5f4
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 51db5ed1-0de3-4db0-8fc8-1706182bb5f4
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
11 ms
|
|
[212320]
|
|
|
424642
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dd9d2237-717b-489e-a3e4-f51ecc36f11c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dd9d2237-717b-489e-a3e4-f51ecc36f11c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212321.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212321.0 (TID 212321) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-7d240550-6b64-4b24-9056-3e937a4ef9a0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212321.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212321.0 (TID 212321) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-7d240550-6b64-4b24-9056-3e937a4ef9a0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424643
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dd9d2237-717b-489e-a3e4-f51ecc36f11c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dd9d2237-717b-489e-a3e4-f51ecc36f11c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
11 ms
|
|
[212321]
|
|
|
424670
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 435fead9-6b29-4ec7-8942-71d7cecbe663
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 435fead9-6b29-4ec7-8942-71d7cecbe663
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212335.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212335.0 (TID 212335) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3f956c41-3d6c-4212-bba3-cf4207b3e85e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212335.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212335.0 (TID 212335) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3f956c41-3d6c-4212-bba3-cf4207b3e85e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424671
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 435fead9-6b29-4ec7-8942-71d7cecbe663
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 435fead9-6b29-4ec7-8942-71d7cecbe663
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
11 ms
|
|
[212335]
|
|
|
424680
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b10c5add-8067-4c01-a8da-5d95f05563be
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b10c5add-8067-4c01-a8da-5d95f05563be
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212340.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212340.0 (TID 212340) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-61a73622-6a63-4ad2-adb4-31a1054e7f71-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212340.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212340.0 (TID 212340) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-61a73622-6a63-4ad2-adb4-31a1054e7f71-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424681
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b10c5add-8067-4c01-a8da-5d95f05563be
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b10c5add-8067-4c01-a8da-5d95f05563be
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
11 ms
|
|
[212340]
|
|
|
424682
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8afa22e2-c546-4b0a-aec7-61487b7c4045
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8afa22e2-c546-4b0a-aec7-61487b7c4045
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212341.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212341.0 (TID 212341) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-61951d22-bba2-43c2-bb92-95b9bcfeaf45-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212341.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212341.0 (TID 212341) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-61951d22-bba2-43c2-bb92-95b9bcfeaf45-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424683
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8afa22e2-c546-4b0a-aec7-61487b7c4045
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8afa22e2-c546-4b0a-aec7-61487b7c4045
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
11 ms
|
|
[212341]
|
|
|
424686
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = bc71414c-f9b4-4c3f-8e55-b34dde044208
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = bc71414c-f9b4-4c3f-8e55-b34dde044208
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:52
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212343.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212343.0 (TID 212343) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6fa6ff70-53d5-4cbb-bd6f-14c1ab270a05-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212343.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212343.0 (TID 212343) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6fa6ff70-53d5-4cbb-bd6f-14c1ab270a05-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424687
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = bc71414c-f9b4-4c3f-8e55-b34dde044208
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = bc71414c-f9b4-4c3f-8e55-b34dde044208
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:52
|
11 ms
|
|
[212343]
|
|
|
424698
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 73042eb5-8e54-4214-af75-5205b679c077
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 73042eb5-8e54-4214-af75-5205b679c077
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:52
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212349.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212349.0 (TID 212349) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-56ece560-f0c6-4d5f-8cab-237b3292ccf6-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212349.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212349.0 (TID 212349) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-56ece560-f0c6-4d5f-8cab-237b3292ccf6-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424699
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 73042eb5-8e54-4214-af75-5205b679c077
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 73042eb5-8e54-4214-af75-5205b679c077
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:52
|
11 ms
|
|
[212349]
|
|
|
424758
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2bf9d487-d2d9-4174-b6a0-476fc82e63bd
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2bf9d487-d2d9-4174-b6a0-476fc82e63bd
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:53
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212379.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212379.0 (TID 212379) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-e1c23329-cccf-428e-b8cc-2cb0b405a658-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212379.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212379.0 (TID 212379) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-e1c23329-cccf-428e-b8cc-2cb0b405a658-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424759
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2bf9d487-d2d9-4174-b6a0-476fc82e63bd
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2bf9d487-d2d9-4174-b6a0-476fc82e63bd
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:53
|
10 ms
|
|
[212379]
|
|
|
424764
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 39b4de21-b562-4fe7-aa5e-d759066eb79b
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 39b4de21-b562-4fe7-aa5e-d759066eb79b
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:53
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212382.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212382.0 (TID 212382) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2704c1a7-5f59-451e-809f-6ce5f71dedb5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212382.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212382.0 (TID 212382) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2704c1a7-5f59-451e-809f-6ce5f71dedb5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424765
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 39b4de21-b562-4fe7-aa5e-d759066eb79b
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 39b4de21-b562-4fe7-aa5e-d759066eb79b
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:53
|
11 ms
|
|
[212382]
|
|
|
424770
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = da04ad9d-569b-403e-bc7e-7b4f8174a141
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = da04ad9d-569b-403e-bc7e-7b4f8174a141
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:53
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212385.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212385.0 (TID 212385) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d6600ef8-87be-4832-85fe-63027d9fa7b3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212385.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212385.0 (TID 212385) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d6600ef8-87be-4832-85fe-63027d9fa7b3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424771
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = da04ad9d-569b-403e-bc7e-7b4f8174a141
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = da04ad9d-569b-403e-bc7e-7b4f8174a141
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:53
|
11 ms
|
|
[212385]
|
|
|
424772
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8ec70eaa-2158-4ebb-9e83-81394995bd70
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8ec70eaa-2158-4ebb-9e83-81394995bd70
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:53
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212386.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212386.0 (TID 212386) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-442c16d6-fb8b-4f05-b7d3-19a4c78870fa-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212386.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212386.0 (TID 212386) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-442c16d6-fb8b-4f05-b7d3-19a4c78870fa-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424773
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8ec70eaa-2158-4ebb-9e83-81394995bd70
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8ec70eaa-2158-4ebb-9e83-81394995bd70
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:53
|
11 ms
|
|
[212386]
|
|
|
424778
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 985d20eb-f7b3-47a4-9cc5-2b2ad11b135d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 985d20eb-f7b3-47a4-9cc5-2b2ad11b135d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:53
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212389.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212389.0 (TID 212389) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-238f2c42-0596-49ff-9f90-459aac95ade1-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212389.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212389.0 (TID 212389) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-238f2c42-0596-49ff-9f90-459aac95ade1-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424779
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 985d20eb-f7b3-47a4-9cc5-2b2ad11b135d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 985d20eb-f7b3-47a4-9cc5-2b2ad11b135d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:53
|
11 ms
|
|
[212389]
|
|
|
424782
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 632e4fd3-4ed8-4fa2-b2df-285ad4d87f9e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 632e4fd3-4ed8-4fa2-b2df-285ad4d87f9e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212391.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212391.0 (TID 212391) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ebb16d80-798b-4a5e-9d6b-fcbae0b56476-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212391.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212391.0 (TID 212391) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ebb16d80-798b-4a5e-9d6b-fcbae0b56476-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424783
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 632e4fd3-4ed8-4fa2-b2df-285ad4d87f9e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 632e4fd3-4ed8-4fa2-b2df-285ad4d87f9e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
11 ms
|
|
[212391]
|
|
|
424784
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fdcad784-6cad-4cfb-8c3e-a79577495dd5
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fdcad784-6cad-4cfb-8c3e-a79577495dd5
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212392.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212392.0 (TID 212392) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f57acbde-7cf3-47c0-9f0c-6eb36ac6b772-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212392.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212392.0 (TID 212392) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f57acbde-7cf3-47c0-9f0c-6eb36ac6b772-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424785
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fdcad784-6cad-4cfb-8c3e-a79577495dd5
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fdcad784-6cad-4cfb-8c3e-a79577495dd5
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
11 ms
|
|
[212392]
|
|
|
424802
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 5087bf68-b169-4062-9934-e7b42cbb2328
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 5087bf68-b169-4062-9934-e7b42cbb2328
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212401.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212401.0 (TID 212401) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3ca29b55-b264-4fae-bfe1-e7a3db141047-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212401.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212401.0 (TID 212401) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3ca29b55-b264-4fae-bfe1-e7a3db141047-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424803
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 5087bf68-b169-4062-9934-e7b42cbb2328
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 5087bf68-b169-4062-9934-e7b42cbb2328
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
11 ms
|
|
[212401]
|
|
|
424804
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b25b47ca-af7f-4cdc-84da-e0973bfa5f61
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b25b47ca-af7f-4cdc-84da-e0973bfa5f61
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212402.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212402.0 (TID 212402) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6492e08c-481c-4f7f-a13b-a87be8c1b001-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212402.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212402.0 (TID 212402) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6492e08c-481c-4f7f-a13b-a87be8c1b001-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424805
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b25b47ca-af7f-4cdc-84da-e0973bfa5f61
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b25b47ca-af7f-4cdc-84da-e0973bfa5f61
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
11 ms
|
|
[212402]
|
|
|
424806
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 5feb7193-fc27-4972-8dc6-e7dbd08af9e2
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 5feb7193-fc27-4972-8dc6-e7dbd08af9e2
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212403.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212403.0 (TID 212403) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5e2ed485-2a65-47a4-a38e-dab84f6b7849-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212403.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212403.0 (TID 212403) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5e2ed485-2a65-47a4-a38e-dab84f6b7849-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424807
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 5feb7193-fc27-4972-8dc6-e7dbd08af9e2
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 5feb7193-fc27-4972-8dc6-e7dbd08af9e2
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
11 ms
|
|
[212403]
|
|
|
424810
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3cc25d09-1f74-4631-9279-178ba4e830c1
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3cc25d09-1f74-4631-9279-178ba4e830c1
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212405.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212405.0 (TID 212405) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-81099137-39b2-47e5-9865-1f8d6b4cc182-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212405.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212405.0 (TID 212405) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-81099137-39b2-47e5-9865-1f8d6b4cc182-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424811
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3cc25d09-1f74-4631-9279-178ba4e830c1
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3cc25d09-1f74-4631-9279-178ba4e830c1
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
11 ms
|
|
[212405]
|
|
|
424822
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b560cb6e-e100-4f81-8c1a-279b391f0c0c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b560cb6e-e100-4f81-8c1a-279b391f0c0c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212411.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212411.0 (TID 212411) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-cfdaf068-e590-4aa8-bef8-9dcc22754c7e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212411.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212411.0 (TID 212411) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-cfdaf068-e590-4aa8-bef8-9dcc22754c7e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424823
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b560cb6e-e100-4f81-8c1a-279b391f0c0c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b560cb6e-e100-4f81-8c1a-279b391f0c0c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
11 ms
|
|
[212411]
|
|
|
424824
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8eeec3cc-3228-43a4-9d13-c15eca1c3090
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8eeec3cc-3228-43a4-9d13-c15eca1c3090
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212412.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212412.0 (TID 212412) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9f66745f-8b78-422a-9aba-d7f9209eef6c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212412.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212412.0 (TID 212412) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9f66745f-8b78-422a-9aba-d7f9209eef6c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424825
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8eeec3cc-3228-43a4-9d13-c15eca1c3090
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8eeec3cc-3228-43a4-9d13-c15eca1c3090
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:54
|
11 ms
|
|
[212412]
|
|
|
424874
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 41bd269d-76c3-4e16-b0c2-d78f02c58770
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 41bd269d-76c3-4e16-b0c2-d78f02c58770
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:56
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212437.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212437.0 (TID 212437) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-893d25e0-36b0-4c8d-9bb6-ee7c482f9251-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212437.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212437.0 (TID 212437) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-893d25e0-36b0-4c8d-9bb6-ee7c482f9251-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424875
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 41bd269d-76c3-4e16-b0c2-d78f02c58770
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 41bd269d-76c3-4e16-b0c2-d78f02c58770
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:56
|
11 ms
|
|
[212437]
|
|
|
424878
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1693a898-20cd-4779-aef8-15fdd01195b1
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1693a898-20cd-4779-aef8-15fdd01195b1
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:56
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212439.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212439.0 (TID 212439) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-603d703f-8dfb-4be2-bb9e-ab0e4e2f6a5c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212439.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212439.0 (TID 212439) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-603d703f-8dfb-4be2-bb9e-ab0e4e2f6a5c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424879
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1693a898-20cd-4779-aef8-15fdd01195b1
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1693a898-20cd-4779-aef8-15fdd01195b1
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:56
|
11 ms
|
|
[212439]
|
|
|
424890
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 80907bce-6f88-4f35-834f-8a60665efc19
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 80907bce-6f88-4f35-834f-8a60665efc19
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:56
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212445.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212445.0 (TID 212445) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-881c5dac-68c4-4b61-9c0b-18b6ec6a0b43-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212445.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212445.0 (TID 212445) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-881c5dac-68c4-4b61-9c0b-18b6ec6a0b43-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424891
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 80907bce-6f88-4f35-834f-8a60665efc19
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 80907bce-6f88-4f35-834f-8a60665efc19
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:56
|
11 ms
|
|
[212445]
|
|
|
424900
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b4a2b1d0-c3c9-41b1-9269-636bca751f12
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b4a2b1d0-c3c9-41b1-9269-636bca751f12
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:56
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212450.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212450.0 (TID 212450) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a2072c21-7bce-4f17-a152-e7683d8fe726-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212450.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212450.0 (TID 212450) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a2072c21-7bce-4f17-a152-e7683d8fe726-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424901
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b4a2b1d0-c3c9-41b1-9269-636bca751f12
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b4a2b1d0-c3c9-41b1-9269-636bca751f12
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:56
|
11 ms
|
|
[212450]
|
|
|
424906
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 5617f5b9-5b8b-4d98-b113-8efee4491ce2
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 5617f5b9-5b8b-4d98-b113-8efee4491ce2
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:56
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212453.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212453.0 (TID 212453) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-da4fdfbb-c7f7-49bf-a29e-67a9ae0236d8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212453.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212453.0 (TID 212453) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-da4fdfbb-c7f7-49bf-a29e-67a9ae0236d8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424907
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 5617f5b9-5b8b-4d98-b113-8efee4491ce2
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 5617f5b9-5b8b-4d98-b113-8efee4491ce2
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:56
|
11 ms
|
|
[212453]
|
|
|
424914
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4fd0493-c740-4a91-b2a9-6ab93868e096
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4fd0493-c740-4a91-b2a9-6ab93868e096
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:56
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212457.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212457.0 (TID 212457) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8258ef56-a9ac-43d7-bf68-617ccadabd6d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212457.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212457.0 (TID 212457) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8258ef56-a9ac-43d7-bf68-617ccadabd6d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424915
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4fd0493-c740-4a91-b2a9-6ab93868e096
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d4fd0493-c740-4a91-b2a9-6ab93868e096
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:56
|
11 ms
|
|
[212457]
|
|
|
424918
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d264a384-b305-4dff-9744-ec4a4457cbc0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d264a384-b305-4dff-9744-ec4a4457cbc0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:56
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212459.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212459.0 (TID 212459) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-22ccf13e-1e57-4097-a53e-28e29133ebcf-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212459.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212459.0 (TID 212459) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-22ccf13e-1e57-4097-a53e-28e29133ebcf-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424919
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d264a384-b305-4dff-9744-ec4a4457cbc0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d264a384-b305-4dff-9744-ec4a4457cbc0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:56
|
11 ms
|
|
[212459]
|
|
|
424932
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8bf7cf86-c8e4-4fe6-a16b-eae64d7f2970
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8bf7cf86-c8e4-4fe6-a16b-eae64d7f2970
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:57
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212466.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212466.0 (TID 212466) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-02d33b2f-3e8a-44ac-a58a-6299ecf88812-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212466.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212466.0 (TID 212466) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-02d33b2f-3e8a-44ac-a58a-6299ecf88812-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424933
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8bf7cf86-c8e4-4fe6-a16b-eae64d7f2970
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8bf7cf86-c8e4-4fe6-a16b-eae64d7f2970
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:57
|
11 ms
|
|
[212466]
|
|
|
424934
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1c9bfd6f-b6d3-475f-bdcc-adb6ea3f4e9d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1c9bfd6f-b6d3-475f-bdcc-adb6ea3f4e9d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:57
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212467.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212467.0 (TID 212467) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-84ddfa66-a417-4d57-a9f2-c03ecca06e52-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212467.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212467.0 (TID 212467) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-84ddfa66-a417-4d57-a9f2-c03ecca06e52-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424935
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1c9bfd6f-b6d3-475f-bdcc-adb6ea3f4e9d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1c9bfd6f-b6d3-475f-bdcc-adb6ea3f4e9d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:57
|
11 ms
|
|
[212467]
|
|
|
424942
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b7b52901-62a0-4890-a1ae-24f33c01f82d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b7b52901-62a0-4890-a1ae-24f33c01f82d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:57
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212471.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212471.0 (TID 212471) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-cb97e0b5-3980-4659-9b41-8a5628fc632c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212471.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212471.0 (TID 212471) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-cb97e0b5-3980-4659-9b41-8a5628fc632c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424943
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b7b52901-62a0-4890-a1ae-24f33c01f82d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b7b52901-62a0-4890-a1ae-24f33c01f82d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:57
|
11 ms
|
|
[212471]
|
|
|
424952
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 543ccd78-239a-4017-8433-5773c45649cb
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 543ccd78-239a-4017-8433-5773c45649cb
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:57
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212476.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212476.0 (TID 212476) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-744488d9-5bbf-49c5-a982-b89da9db3993-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212476.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212476.0 (TID 212476) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-744488d9-5bbf-49c5-a982-b89da9db3993-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424953
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 543ccd78-239a-4017-8433-5773c45649cb
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 543ccd78-239a-4017-8433-5773c45649cb
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:57
|
11 ms
|
|
[212476]
|
|
|
425018
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ab2ad061-767d-42cb-ae22-181b58c9504c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ab2ad061-767d-42cb-ae22-181b58c9504c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:59
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212509.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212509.0 (TID 212509) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c94d8c40-e135-48d4-bf87-696394dced35-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212509.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212509.0 (TID 212509) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c94d8c40-e135-48d4-bf87-696394dced35-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
425019
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ab2ad061-767d-42cb-ae22-181b58c9504c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ab2ad061-767d-42cb-ae22-181b58c9504c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:59
|
11 ms
|
|
[212509]
|
|
|
425034
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cafe93d2-7a2d-474d-929e-de49ac5f0223
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cafe93d2-7a2d-474d-929e-de49ac5f0223
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:59
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212517.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212517.0 (TID 212517) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-0833b709-7e76-41ae-8211-ec7fcdee4167-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212517.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212517.0 (TID 212517) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-0833b709-7e76-41ae-8211-ec7fcdee4167-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
425035
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cafe93d2-7a2d-474d-929e-de49ac5f0223
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cafe93d2-7a2d-474d-929e-de49ac5f0223
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:59
|
11 ms
|
|
[212517]
|
|
|
425040
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = c2f9f74f-d60d-4621-a3c4-b6e68b2f6cf6
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = c2f9f74f-d60d-4621-a3c4-b6e68b2f6cf6
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:59
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212520.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212520.0 (TID 212520) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9bd91b71-4266-461e-be15-920d92b49b92-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212520.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212520.0 (TID 212520) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9bd91b71-4266-461e-be15-920d92b49b92-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
425041
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = c2f9f74f-d60d-4621-a3c4-b6e68b2f6cf6
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = c2f9f74f-d60d-4621-a3c4-b6e68b2f6cf6
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:59
|
11 ms
|
|
[212520]
|
|
|
425042
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 4df8c0a1-ce7c-4ba8-9568-7096fe4c04e6
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 4df8c0a1-ce7c-4ba8-9568-7096fe4c04e6
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:59
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212521.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212521.0 (TID 212521) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ebbd6340-773f-4708-9f3c-a005ea5981b4-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212521.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212521.0 (TID 212521) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ebbd6340-773f-4708-9f3c-a005ea5981b4-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
425043
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 4df8c0a1-ce7c-4ba8-9568-7096fe4c04e6
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 4df8c0a1-ce7c-4ba8-9568-7096fe4c04e6
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:59
|
11 ms
|
|
[212521]
|
|
|
425050
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c81df14-55dc-4205-9ca2-6b49184ca854
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c81df14-55dc-4205-9ca2-6b49184ca854
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:59
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212525.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212525.0 (TID 212525) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d9566ed7-26ee-4c05-8a20-efcfa9c0a555-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212525.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212525.0 (TID 212525) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d9566ed7-26ee-4c05-8a20-efcfa9c0a555-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
425051
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c81df14-55dc-4205-9ca2-6b49184ca854
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c81df14-55dc-4205-9ca2-6b49184ca854
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:59
|
11 ms
|
|
[212525]
|
|
|
425210
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 067b6f7a-d48f-4f57-a599-8ce8df884205
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 067b6f7a-d48f-4f57-a599-8ce8df884205
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:05:03
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212605.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212605.0 (TID 212605) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-992ae10a-522e-4c7c-92da-d7e93320f5d5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212605.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212605.0 (TID 212605) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-992ae10a-522e-4c7c-92da-d7e93320f5d5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
425211
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 067b6f7a-d48f-4f57-a599-8ce8df884205
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 067b6f7a-d48f-4f57-a599-8ce8df884205
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:05:03
|
11 ms
|
|
[212605]
|
|
|
425214
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dda2a38b-d02e-47da-b610-9600682ea538
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dda2a38b-d02e-47da-b610-9600682ea538
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:05:03
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212607.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212607.0 (TID 212607) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-28242f1f-3c87-4f3b-8521-019924902f0c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212607.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212607.0 (TID 212607) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-28242f1f-3c87-4f3b-8521-019924902f0c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
425215
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dda2a38b-d02e-47da-b610-9600682ea538
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = dda2a38b-d02e-47da-b610-9600682ea538
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:05:03
|
11 ms
|
|
[212607]
|
|
|
425220
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 26ed0750-05d9-4622-8d05-01d5af94896f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 26ed0750-05d9-4622-8d05-01d5af94896f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:05:03
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212610.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212610.0 (TID 212610) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a3d4084b-c7a6-4182-8c0d-cb889c5320b9-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212610.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212610.0 (TID 212610) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a3d4084b-c7a6-4182-8c0d-cb889c5320b9-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
425221
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 26ed0750-05d9-4622-8d05-01d5af94896f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 26ed0750-05d9-4622-8d05-01d5af94896f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:05:03
|
11 ms
|
|
[212610]
|
|
|
425296
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = c1d71510-3d79-4455-afe0-4120a7bca00e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = c1d71510-3d79-4455-afe0-4120a7bca00e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:05:05
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212648.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212648.0 (TID 212648) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6b82b415-c0ea-4c66-93d1-4265a32b8f1c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212648.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212648.0 (TID 212648) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6b82b415-c0ea-4c66-93d1-4265a32b8f1c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
425297
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = c1d71510-3d79-4455-afe0-4120a7bca00e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = c1d71510-3d79-4455-afe0-4120a7bca00e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:05:05
|
11 ms
|
|
[212648]
|
|
|
425342
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c9a887b-d115-44f9-9467-f74a092ddda1
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c9a887b-d115-44f9-9467-f74a092ddda1
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:05:06
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212671.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212671.0 (TID 212671) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5a6af797-2e19-4384-8df5-ff8c7324c2bc-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212671.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212671.0 (TID 212671) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5a6af797-2e19-4384-8df5-ff8c7324c2bc-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
425343
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c9a887b-d115-44f9-9467-f74a092ddda1
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c9a887b-d115-44f9-9467-f74a092ddda1
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:05:06
|
11 ms
|
|
[212671]
|
|
|
424384
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 030375ec-dfc7-4931-b599-c5532863e193
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 030375ec-dfc7-4931-b599-c5532863e193
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:45
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212192.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212192.0 (TID 212192) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ad2db132-e662-4977-8821-b562609ad26f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212192.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212192.0 (TID 212192) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ad2db132-e662-4977-8821-b562609ad26f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424385
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 030375ec-dfc7-4931-b599-c5532863e193
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 030375ec-dfc7-4931-b599-c5532863e193
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:45
|
12 ms
|
|
[212192]
|
|
|
424390
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 69c469a8-45cd-43f1-970a-37acc639bfda
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 69c469a8-45cd-43f1-970a-37acc639bfda
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:45
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212195.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212195.0 (TID 212195) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-61562a04-500f-403c-a787-2f0daa62a7d1-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212195.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212195.0 (TID 212195) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-61562a04-500f-403c-a787-2f0daa62a7d1-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424391
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 69c469a8-45cd-43f1-970a-37acc639bfda
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 69c469a8-45cd-43f1-970a-37acc639bfda
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:45
|
11 ms
|
|
[212195]
|
|
|
424392
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 9e641434-d850-401e-80e8-1d47dab63525
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 9e641434-d850-401e-80e8-1d47dab63525
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:45
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212196.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212196.0 (TID 212196) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a7fcc5a1-ed7c-4177-8f20-6e8cf14ab1ac-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212196.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212196.0 (TID 212196) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a7fcc5a1-ed7c-4177-8f20-6e8cf14ab1ac-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424393
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 9e641434-d850-401e-80e8-1d47dab63525
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 9e641434-d850-401e-80e8-1d47dab63525
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:45
|
11 ms
|
|
[212196]
|
|
|
424394
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d68f9a5d-c791-4be6-abb1-01ca9d83bc90
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d68f9a5d-c791-4be6-abb1-01ca9d83bc90
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:45
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212197.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212197.0 (TID 212197) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-75021193-d491-46ca-b5dd-5c53df2f0485-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212197.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212197.0 (TID 212197) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-75021193-d491-46ca-b5dd-5c53df2f0485-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424395
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d68f9a5d-c791-4be6-abb1-01ca9d83bc90
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d68f9a5d-c791-4be6-abb1-01ca9d83bc90
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:45
|
11 ms
|
|
[212197]
|
|
|
424436
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 7f103a42-31e3-4957-8401-0709124f5e74
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 7f103a42-31e3-4957-8401-0709124f5e74
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:46
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212218.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212218.0 (TID 212218) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-04f43c96-75d6-4766-bdaf-c2fa98dafdd1-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212218.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212218.0 (TID 212218) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-04f43c96-75d6-4766-bdaf-c2fa98dafdd1-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424437
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 7f103a42-31e3-4957-8401-0709124f5e74
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 7f103a42-31e3-4957-8401-0709124f5e74
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:46
|
11 ms
|
|
[212218]
|
|
|
424438
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = de03facc-5f9b-45d6-b28b-5aafdd60e3ad
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = de03facc-5f9b-45d6-b28b-5aafdd60e3ad
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:46
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212219.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212219.0 (TID 212219) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-66be5fc9-a573-4ab1-83e4-2b1e8f42f60d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212219.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212219.0 (TID 212219) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-66be5fc9-a573-4ab1-83e4-2b1e8f42f60d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424439
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = de03facc-5f9b-45d6-b28b-5aafdd60e3ad
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = de03facc-5f9b-45d6-b28b-5aafdd60e3ad
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:46
|
11 ms
|
|
[212219]
|
|
|
424450
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 6143ea2c-4b66-4838-80c1-8f45edac39d2
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 6143ea2c-4b66-4838-80c1-8f45edac39d2
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212225.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212225.0 (TID 212225) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f68bce05-b0a2-4289-8d57-4ccec4ac4b72-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212225.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212225.0 (TID 212225) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f68bce05-b0a2-4289-8d57-4ccec4ac4b72-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424451
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 6143ea2c-4b66-4838-80c1-8f45edac39d2
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 6143ea2c-4b66-4838-80c1-8f45edac39d2
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
11 ms
|
|
[212225]
|
|
|
424454
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 6aa80ab3-1ad4-4a46-bd4d-24b7c7784d75
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 6aa80ab3-1ad4-4a46-bd4d-24b7c7784d75
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212227.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212227.0 (TID 212227) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b622b4ea-3eda-4d09-b5c3-84979cdd544a-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212227.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212227.0 (TID 212227) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b622b4ea-3eda-4d09-b5c3-84979cdd544a-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424455
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 6aa80ab3-1ad4-4a46-bd4d-24b7c7784d75
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 6aa80ab3-1ad4-4a46-bd4d-24b7c7784d75
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
11 ms
|
|
[212227]
|
|
|
424456
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = eae08e34-d675-428a-9209-3a6744271c6e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = eae08e34-d675-428a-9209-3a6744271c6e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212228.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212228.0 (TID 212228) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2975f354-586e-497f-bd93-ee617f8a369c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212228.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212228.0 (TID 212228) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2975f354-586e-497f-bd93-ee617f8a369c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424457
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = eae08e34-d675-428a-9209-3a6744271c6e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = eae08e34-d675-428a-9209-3a6744271c6e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
11 ms
|
|
[212228]
|
|
|
424458
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1999e2e-8700-4cbf-9c6e-3a420e61f23a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1999e2e-8700-4cbf-9c6e-3a420e61f23a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212229.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212229.0 (TID 212229) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-33c5156b-ef3c-47b1-b868-1fcbcd1b8bde-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212229.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212229.0 (TID 212229) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-33c5156b-ef3c-47b1-b868-1fcbcd1b8bde-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424459
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1999e2e-8700-4cbf-9c6e-3a420e61f23a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = e1999e2e-8700-4cbf-9c6e-3a420e61f23a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
11 ms
|
|
[212229]
|
|
|
424460
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3c7340b5-6e9c-4e00-a64b-0fb05acba235
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3c7340b5-6e9c-4e00-a64b-0fb05acba235
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212230.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212230.0 (TID 212230) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5397e48f-5eda-4f54-974a-4666bafa2e17-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212230.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212230.0 (TID 212230) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5397e48f-5eda-4f54-974a-4666bafa2e17-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424461
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3c7340b5-6e9c-4e00-a64b-0fb05acba235
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3c7340b5-6e9c-4e00-a64b-0fb05acba235
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
11 ms
|
|
[212230]
|
|
|
424462
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f18e2842-2b9f-4fa0-bc26-2c41025bd6f3
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f18e2842-2b9f-4fa0-bc26-2c41025bd6f3
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212231.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212231.0 (TID 212231) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-94d25253-1831-4606-bb4a-c3848f58abbe-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212231.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212231.0 (TID 212231) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-94d25253-1831-4606-bb4a-c3848f58abbe-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424463
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f18e2842-2b9f-4fa0-bc26-2c41025bd6f3
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = f18e2842-2b9f-4fa0-bc26-2c41025bd6f3
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
12 ms
|
|
[212231]
|
|
|
424466
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 748d4f95-4056-4052-acf8-789ce0e65ece
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 748d4f95-4056-4052-acf8-789ce0e65ece
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212233.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212233.0 (TID 212233) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-45080cc5-d46c-4d3c-a4a1-eba0d3863664-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212233.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212233.0 (TID 212233) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-45080cc5-d46c-4d3c-a4a1-eba0d3863664-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424467
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 748d4f95-4056-4052-acf8-789ce0e65ece
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 748d4f95-4056-4052-acf8-789ce0e65ece
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
11 ms
|
|
[212233]
|
|
|
424476
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fd16799e-bec3-42ee-9761-7da4c9737536
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fd16799e-bec3-42ee-9761-7da4c9737536
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212238.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212238.0 (TID 212238) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9988badf-de1c-453a-857b-f34918023c22-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212238.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212238.0 (TID 212238) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9988badf-de1c-453a-857b-f34918023c22-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424477
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fd16799e-bec3-42ee-9761-7da4c9737536
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = fd16799e-bec3-42ee-9761-7da4c9737536
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
11 ms
|
|
[212238]
|
|
|
424478
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b67091fb-3a14-4c67-8bdf-08ed07d940ba
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b67091fb-3a14-4c67-8bdf-08ed07d940ba
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212239.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212239.0 (TID 212239) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-946579d6-3016-4dcc-af56-a88d8f9c725f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212239.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212239.0 (TID 212239) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-946579d6-3016-4dcc-af56-a88d8f9c725f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424479
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b67091fb-3a14-4c67-8bdf-08ed07d940ba
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b67091fb-3a14-4c67-8bdf-08ed07d940ba
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
11 ms
|
|
[212239]
|
|
|
424482
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ef0e1140-bc04-41da-9939-316e12f78997
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ef0e1140-bc04-41da-9939-316e12f78997
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212241.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212241.0 (TID 212241) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-948f0598-e108-41d6-bf75-8fac63090296-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212241.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212241.0 (TID 212241) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-948f0598-e108-41d6-bf75-8fac63090296-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424483
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ef0e1140-bc04-41da-9939-316e12f78997
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = ef0e1140-bc04-41da-9939-316e12f78997
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
11 ms
|
|
[212241]
|
|
|
424484
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = a01053a3-b632-4b22-812a-8cf166a0bd44
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = a01053a3-b632-4b22-812a-8cf166a0bd44
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212242.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212242.0 (TID 212242) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-63e0b25a-1d34-4b15-a339-3466d2843862-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212242.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212242.0 (TID 212242) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-63e0b25a-1d34-4b15-a339-3466d2843862-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424485
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = a01053a3-b632-4b22-812a-8cf166a0bd44
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = a01053a3-b632-4b22-812a-8cf166a0bd44
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:47
|
11 ms
|
|
[212242]
|
|
|
424500
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 7c3d7013-6c09-4eed-9aab-38a2828d8498
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 7c3d7013-6c09-4eed-9aab-38a2828d8498
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212250.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212250.0 (TID 212250) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-fe009366-c6dd-480d-b4a2-c466db0fbbb4-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212250.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212250.0 (TID 212250) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-fe009366-c6dd-480d-b4a2-c466db0fbbb4-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424501
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 7c3d7013-6c09-4eed-9aab-38a2828d8498
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 7c3d7013-6c09-4eed-9aab-38a2828d8498
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
12 ms
|
|
[212250]
|
|
|
424530
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0e8b5303-1801-45a1-b854-b1c346792441
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0e8b5303-1801-45a1-b854-b1c346792441
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212265.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212265.0 (TID 212265) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-e646570e-c378-4562-b1e4-25a51903cbd5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212265.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212265.0 (TID 212265) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-e646570e-c378-4562-b1e4-25a51903cbd5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424531
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0e8b5303-1801-45a1-b854-b1c346792441
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0e8b5303-1801-45a1-b854-b1c346792441
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
11 ms
|
|
[212265]
|
|
|
424532
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15bc81db-d2fa-4f30-b250-f80ec30cea4d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15bc81db-d2fa-4f30-b250-f80ec30cea4d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212266.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212266.0 (TID 212266) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-94100587-db16-4198-9e2a-e2da8b3a1c75-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212266.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212266.0 (TID 212266) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-94100587-db16-4198-9e2a-e2da8b3a1c75-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424533
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15bc81db-d2fa-4f30-b250-f80ec30cea4d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15bc81db-d2fa-4f30-b250-f80ec30cea4d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
11 ms
|
|
[212266]
|
|
|
424536
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d021f24d-61cf-4e47-9753-8a8e0a8936d8
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d021f24d-61cf-4e47-9753-8a8e0a8936d8
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212268.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212268.0 (TID 212268) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-17f619a7-6ed9-4d4c-955d-69cd82e9fc82-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212268.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212268.0 (TID 212268) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-17f619a7-6ed9-4d4c-955d-69cd82e9fc82-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424537
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d021f24d-61cf-4e47-9753-8a8e0a8936d8
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d021f24d-61cf-4e47-9753-8a8e0a8936d8
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
10 ms
|
|
[212268]
|
|
|
424538
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 800ed3ea-6cbc-4adc-bc3f-e459bca06ca9
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 800ed3ea-6cbc-4adc-bc3f-e459bca06ca9
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212269.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212269.0 (TID 212269) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2b7047a2-db3a-4b35-abff-8aa6cf2a3400-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212269.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212269.0 (TID 212269) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2b7047a2-db3a-4b35-abff-8aa6cf2a3400-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424539
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 800ed3ea-6cbc-4adc-bc3f-e459bca06ca9
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 800ed3ea-6cbc-4adc-bc3f-e459bca06ca9
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
11 ms
|
|
[212269]
|
|
|
424540
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d89169a0-917f-4d70-99d6-a25d6fa6a042
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d89169a0-917f-4d70-99d6-a25d6fa6a042
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212270.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212270.0 (TID 212270) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c00a5b6b-e323-4b1b-93d1-e77fa80c8979-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212270.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212270.0 (TID 212270) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c00a5b6b-e323-4b1b-93d1-e77fa80c8979-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424541
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d89169a0-917f-4d70-99d6-a25d6fa6a042
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = d89169a0-917f-4d70-99d6-a25d6fa6a042
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:48
|
11 ms
|
|
[212270]
|
|
|
424546
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b312a8db-78b6-4d81-a42f-a63e454c2eab
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b312a8db-78b6-4d81-a42f-a63e454c2eab
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212273.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212273.0 (TID 212273) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-4f7fc7b4-a1ad-4ec1-9ff6-6bbda2a09199-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212273.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212273.0 (TID 212273) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-4f7fc7b4-a1ad-4ec1-9ff6-6bbda2a09199-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424547
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b312a8db-78b6-4d81-a42f-a63e454c2eab
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b312a8db-78b6-4d81-a42f-a63e454c2eab
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212273]
|
|
|
424552
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 94f5e144-d55b-4032-b4a0-f93bbac06871
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 94f5e144-d55b-4032-b4a0-f93bbac06871
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212276.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212276.0 (TID 212276) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b69e5988-0bd1-4005-ac9f-893df5b1576e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212276.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212276.0 (TID 212276) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b69e5988-0bd1-4005-ac9f-893df5b1576e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424553
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 94f5e144-d55b-4032-b4a0-f93bbac06871
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 94f5e144-d55b-4032-b4a0-f93bbac06871
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
12 ms
|
|
[212276]
|
|
|
424558
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8258b84c-f189-4e70-8adf-bea3c7ffadac
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8258b84c-f189-4e70-8adf-bea3c7ffadac
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212279.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212279.0 (TID 212279) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-bd09a01f-291e-42bb-bbef-00f8ec820bc2-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212279.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212279.0 (TID 212279) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-bd09a01f-291e-42bb-bbef-00f8ec820bc2-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424559
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8258b84c-f189-4e70-8adf-bea3c7ffadac
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 8258b84c-f189-4e70-8adf-bea3c7ffadac
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
12 ms
|
|
[212279]
|
|
|
424562
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 40ba4a46-7d85-4120-965d-1d6dc51cf416
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 40ba4a46-7d85-4120-965d-1d6dc51cf416
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212281.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212281.0 (TID 212281) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1f41df1e-afea-4a63-9f82-62a0d43cf71e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212281.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212281.0 (TID 212281) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1f41df1e-afea-4a63-9f82-62a0d43cf71e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424563
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 40ba4a46-7d85-4120-965d-1d6dc51cf416
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 40ba4a46-7d85-4120-965d-1d6dc51cf416
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212281]
|
|
|
424564
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 4791e249-8911-4d2b-9d0f-bd4b88a1e2dc
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 4791e249-8911-4d2b-9d0f-bd4b88a1e2dc
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212282.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212282.0 (TID 212282) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-15dc0411-e1ca-4421-b360-757b5a47357d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212282.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212282.0 (TID 212282) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-15dc0411-e1ca-4421-b360-757b5a47357d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424565
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 4791e249-8911-4d2b-9d0f-bd4b88a1e2dc
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 4791e249-8911-4d2b-9d0f-bd4b88a1e2dc
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212282]
|
|
|
424570
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = af198069-8e07-499a-86df-cd3ee5839778
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = af198069-8e07-499a-86df-cd3ee5839778
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212285.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212285.0 (TID 212285) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-45afe6cf-d6a1-4c13-be1e-20f8100f3e82-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212285.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212285.0 (TID 212285) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-45afe6cf-d6a1-4c13-be1e-20f8100f3e82-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424571
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = af198069-8e07-499a-86df-cd3ee5839778
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = af198069-8e07-499a-86df-cd3ee5839778
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212285]
|
|
|
424580
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cd80db7b-ae7a-4f37-9c78-bbe9646c39de
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cd80db7b-ae7a-4f37-9c78-bbe9646c39de
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212290.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212290.0 (TID 212290) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-0271eb2c-786b-43ed-a416-fd280b36bed4-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212290.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212290.0 (TID 212290) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-0271eb2c-786b-43ed-a416-fd280b36bed4-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424581
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cd80db7b-ae7a-4f37-9c78-bbe9646c39de
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cd80db7b-ae7a-4f37-9c78-bbe9646c39de
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212290]
|
|
|
424582
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 7d59a6b0-0350-46fa-94ac-924f6e18485b
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 7d59a6b0-0350-46fa-94ac-924f6e18485b
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212291.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212291.0 (TID 212291) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-4394ea29-52a1-4e9d-a9b3-8014f5bc9fd3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212291.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212291.0 (TID 212291) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-4394ea29-52a1-4e9d-a9b3-8014f5bc9fd3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424583
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 7d59a6b0-0350-46fa-94ac-924f6e18485b
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 7d59a6b0-0350-46fa-94ac-924f6e18485b
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212291]
|
|
|
424584
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 33834efb-dc57-4158-b8a9-56792933cf0e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 33834efb-dc57-4158-b8a9-56792933cf0e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212292.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212292.0 (TID 212292) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2f91b0cc-5d29-4751-b4c3-9b243d4f04a5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212292.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212292.0 (TID 212292) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2f91b0cc-5d29-4751-b4c3-9b243d4f04a5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424585
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 33834efb-dc57-4158-b8a9-56792933cf0e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 33834efb-dc57-4158-b8a9-56792933cf0e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212292]
|
|
|
424586
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 09d3271e-c32f-4b9f-815d-b11f7979ea4b
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 09d3271e-c32f-4b9f-815d-b11f7979ea4b
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212293.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212293.0 (TID 212293) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ee673bec-accb-47b7-866f-7f5cfbf94803-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212293.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212293.0 (TID 212293) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ee673bec-accb-47b7-866f-7f5cfbf94803-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424587
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 09d3271e-c32f-4b9f-815d-b11f7979ea4b
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 09d3271e-c32f-4b9f-815d-b11f7979ea4b
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:49
|
11 ms
|
|
[212293]
|
|
|
424592
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b1c7da63-3e39-494d-a90c-0c5e857be782
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b1c7da63-3e39-494d-a90c-0c5e857be782
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:50
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212296.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212296.0 (TID 212296) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ceec1163-411c-413e-bd13-c1c5f64abe84-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212296.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212296.0 (TID 212296) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ceec1163-411c-413e-bd13-c1c5f64abe84-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424593
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b1c7da63-3e39-494d-a90c-0c5e857be782
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = b1c7da63-3e39-494d-a90c-0c5e857be782
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:50
|
12 ms
|
|
[212296]
|
|
|
424596
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cbf0c86a-1c4f-444f-b84e-30645bc379c4
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cbf0c86a-1c4f-444f-b84e-30645bc379c4
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:50
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212298.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212298.0 (TID 212298) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-13d0124c-8f32-4241-8e5a-cb46bf8d1ba3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212298.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212298.0 (TID 212298) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-13d0124c-8f32-4241-8e5a-cb46bf8d1ba3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424597
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cbf0c86a-1c4f-444f-b84e-30645bc379c4
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = cbf0c86a-1c4f-444f-b84e-30645bc379c4
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:50
|
11 ms
|
|
[212298]
|
|
|
424650
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 106f6fc7-0c3b-4c2e-b3d8-6522154bb1db
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 106f6fc7-0c3b-4c2e-b3d8-6522154bb1db
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212325.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212325.0 (TID 212325) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-fb66c55c-ad17-442d-aed0-8011e0e3e99f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212325.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212325.0 (TID 212325) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-fb66c55c-ad17-442d-aed0-8011e0e3e99f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424651
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 106f6fc7-0c3b-4c2e-b3d8-6522154bb1db
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 106f6fc7-0c3b-4c2e-b3d8-6522154bb1db
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
11 ms
|
|
[212325]
|
|
|
424654
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0ccf8553-37cb-4e36-aba5-f993ac601362
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0ccf8553-37cb-4e36-aba5-f993ac601362
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212327.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212327.0 (TID 212327) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-748c5dc9-623e-4660-a102-e1a95deecf23-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212327.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212327.0 (TID 212327) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-748c5dc9-623e-4660-a102-e1a95deecf23-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424655
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0ccf8553-37cb-4e36-aba5-f993ac601362
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0ccf8553-37cb-4e36-aba5-f993ac601362
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
11 ms
|
|
[212327]
|
|
|
424656
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 9186022f-8005-4448-9239-81cb6147c6b0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 9186022f-8005-4448-9239-81cb6147c6b0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212328.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212328.0 (TID 212328) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2b18e389-81b6-4faf-9afe-176169a2e39f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212328.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212328.0 (TID 212328) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2b18e389-81b6-4faf-9afe-176169a2e39f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424657
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 9186022f-8005-4448-9239-81cb6147c6b0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 9186022f-8005-4448-9239-81cb6147c6b0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
11 ms
|
|
[212328]
|
|
|
424660
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 56514c89-fac0-40cc-995f-bfb503ff7e01
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 56514c89-fac0-40cc-995f-bfb503ff7e01
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212330.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212330.0 (TID 212330) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-552266c7-aa2a-4467-b862-64d6c88428a9-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212330.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212330.0 (TID 212330) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-552266c7-aa2a-4467-b862-64d6c88428a9-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424661
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 56514c89-fac0-40cc-995f-bfb503ff7e01
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 56514c89-fac0-40cc-995f-bfb503ff7e01
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
12 ms
|
|
[212330]
|
|
|
424662
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = c2a5c8b9-e250-4fe3-8b67-feed5da9359c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = c2a5c8b9-e250-4fe3-8b67-feed5da9359c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212331.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212331.0 (TID 212331) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ce91ff6b-4b87-4cd5-92e1-7dd1ad86c54b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212331.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212331.0 (TID 212331) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ce91ff6b-4b87-4cd5-92e1-7dd1ad86c54b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424663
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = c2a5c8b9-e250-4fe3-8b67-feed5da9359c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = c2a5c8b9-e250-4fe3-8b67-feed5da9359c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
11 ms
|
|
[212331]
|
|
|
424664
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 495ec88d-4c76-4b00-9e98-caaada540a9e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 495ec88d-4c76-4b00-9e98-caaada540a9e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 212332.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212332.0 (TID 212332) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-4b4ceefc-05e0-4a1a-8ec6-90e674de2fe2-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 212332.0 failed 1 times, most recent failure: Lost task 0.0 in stage 212332.0 (TID 212332) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-4b4ceefc-05e0-4a1a-8ec6-90e674de2fe2-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
424665
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 495ec88d-4c76-4b00-9e98-caaada540a9e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 495ec88d-4c76-4b00-9e98-caaada540a9e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 12:04:51
|
12 ms
|
|
[212332]
|
|