|
697750
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 00152be6-d580-433f-aed9-0426d6ede889
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 00152be6-d580-433f-aed9-0426d6ede889
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:17
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348875.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348875.0 (TID 348875) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-58ba603e-3d64-4abb-8f5a-472e0b99d801-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348875.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348875.0 (TID 348875) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-58ba603e-3d64-4abb-8f5a-472e0b99d801-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697751
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 00152be6-d580-433f-aed9-0426d6ede889
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 00152be6-d580-433f-aed9-0426d6ede889
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:17
|
13 ms
|
|
[348875]
|
|
|
697624
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 00310494-518e-45c9-8a89-9dcee6f6835f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 00310494-518e-45c9-8a89-9dcee6f6835f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:14
|
19 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348812.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348812.0 (TID 348812) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1406a963-1121-46c9-bf0a-466e11523a7f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348812.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348812.0 (TID 348812) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1406a963-1121-46c9-bf0a-466e11523a7f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697625
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 00310494-518e-45c9-8a89-9dcee6f6835f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 00310494-518e-45c9-8a89-9dcee6f6835f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:14
|
15 ms
|
|
[348812]
|
|
|
698054
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 009df63a-11cb-4f5f-8f7a-dc126cbc7eea
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 009df63a-11cb-4f5f-8f7a-dc126cbc7eea
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:24
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349027.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349027.0 (TID 349027) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-28012a4a-b5d4-4a65-9637-ff136443a33c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349027.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349027.0 (TID 349027) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-28012a4a-b5d4-4a65-9637-ff136443a33c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698055
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 009df63a-11cb-4f5f-8f7a-dc126cbc7eea
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 009df63a-11cb-4f5f-8f7a-dc126cbc7eea
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:24
|
10 ms
|
|
[349027]
|
|
|
697672
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 00dd1a4e-5fd1-4fb6-a0c8-a393f09d1c09
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 00dd1a4e-5fd1-4fb6-a0c8-a393f09d1c09
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:16
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348836.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348836.0 (TID 348836) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d1b158bd-c016-42f4-ab5d-51e93842739d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348836.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348836.0 (TID 348836) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d1b158bd-c016-42f4-ab5d-51e93842739d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697673
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 00dd1a4e-5fd1-4fb6-a0c8-a393f09d1c09
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 00dd1a4e-5fd1-4fb6-a0c8-a393f09d1c09
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:16
|
13 ms
|
|
[348836]
|
|
|
698072
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 01a269bf-b473-444e-a00a-3f7da24d9895
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 01a269bf-b473-444e-a00a-3f7da24d9895
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:24
|
21 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349036.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349036.0 (TID 349036) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-67dd07f9-0c1a-4303-be98-3d6ae2c8524d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349036.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349036.0 (TID 349036) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-67dd07f9-0c1a-4303-be98-3d6ae2c8524d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698073
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 01a269bf-b473-444e-a00a-3f7da24d9895
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 01a269bf-b473-444e-a00a-3f7da24d9895
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:24
|
18 ms
|
|
[349036]
|
|
|
697946
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 023dc220-0c20-4602-837a-bde230e98b50
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 023dc220-0c20-4602-837a-bde230e98b50
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:22
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348973.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348973.0 (TID 348973) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-78941a96-6133-494d-a1d7-0a3970caac36-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348973.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348973.0 (TID 348973) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-78941a96-6133-494d-a1d7-0a3970caac36-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697947
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 023dc220-0c20-4602-837a-bde230e98b50
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 023dc220-0c20-4602-837a-bde230e98b50
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:22
|
14 ms
|
|
[348973]
|
|
|
698418
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 027dd9fe-b02c-4e0d-a8c5-e5f7e3676e4d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 027dd9fe-b02c-4e0d-a8c5-e5f7e3676e4d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:33
|
20 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349209.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349209.0 (TID 349209) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-412835ac-0ed6-4a92-b4aa-d6d049189ab4-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349209.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349209.0 (TID 349209) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-412835ac-0ed6-4a92-b4aa-d6d049189ab4-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698419
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 027dd9fe-b02c-4e0d-a8c5-e5f7e3676e4d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 027dd9fe-b02c-4e0d-a8c5-e5f7e3676e4d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:33
|
16 ms
|
|
[349209]
|
|
|
697590
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 028f2d39-b23d-4831-88d8-0ef8dbb6bf81
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 028f2d39-b23d-4831-88d8-0ef8dbb6bf81
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:14
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348795.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348795.0 (TID 348795) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b936e6e1-a7f7-4538-91f4-b537c3b39668-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348795.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348795.0 (TID 348795) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b936e6e1-a7f7-4538-91f4-b537c3b39668-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697591
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 028f2d39-b23d-4831-88d8-0ef8dbb6bf81
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 028f2d39-b23d-4831-88d8-0ef8dbb6bf81
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:14
|
11 ms
|
|
[348795]
|
|
|
698076
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 02e4fc62-ecfd-45b9-87ff-ffe58e8cb02c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 02e4fc62-ecfd-45b9-87ff-ffe58e8cb02c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:25
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349038.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349038.0 (TID 349038) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5aa7c53f-0546-44d3-847b-dd0d3b42f19f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349038.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349038.0 (TID 349038) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5aa7c53f-0546-44d3-847b-dd0d3b42f19f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698077
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 02e4fc62-ecfd-45b9-87ff-ffe58e8cb02c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 02e4fc62-ecfd-45b9-87ff-ffe58e8cb02c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:25
|
13 ms
|
|
[349038]
|
|
|
697542
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 03882e09-0e17-4a51-a362-b67777f3fba3
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 03882e09-0e17-4a51-a362-b67777f3fba3
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:13
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348771.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348771.0 (TID 348771) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2bbc551d-6cd3-49e2-867f-cbc6d1360f1e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348771.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348771.0 (TID 348771) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2bbc551d-6cd3-49e2-867f-cbc6d1360f1e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697543
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 03882e09-0e17-4a51-a362-b67777f3fba3
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 03882e09-0e17-4a51-a362-b67777f3fba3
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:13
|
14 ms
|
|
[348771]
|
|
|
697780
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 043078dd-9fe0-43ea-b840-e9894a673b14
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 043078dd-9fe0-43ea-b840-e9894a673b14
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:18
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348890.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348890.0 (TID 348890) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f100f15d-526b-444c-8aae-8a0d8e529d80-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348890.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348890.0 (TID 348890) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f100f15d-526b-444c-8aae-8a0d8e529d80-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697781
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 043078dd-9fe0-43ea-b840-e9894a673b14
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 043078dd-9fe0-43ea-b840-e9894a673b14
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:18
|
11 ms
|
|
[348890]
|
|
|
698284
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 050754eb-c597-4466-97ba-4d39dc9275fa
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 050754eb-c597-4466-97ba-4d39dc9275fa
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:29
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349142.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349142.0 (TID 349142) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f9925ff1-8acf-4aa5-b113-a8a0ef61d7e3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349142.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349142.0 (TID 349142) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f9925ff1-8acf-4aa5-b113-a8a0ef61d7e3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698285
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 050754eb-c597-4466-97ba-4d39dc9275fa
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 050754eb-c597-4466-97ba-4d39dc9275fa
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:29
|
12 ms
|
|
[349142]
|
|
|
697838
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0620fb9b-08a6-4de8-8eb8-45cb7b84f75a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0620fb9b-08a6-4de8-8eb8-45cb7b84f75a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:19
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348919.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348919.0 (TID 348919) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-03ad8332-7466-467b-8666-c09c6bc728d4-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348919.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348919.0 (TID 348919) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-03ad8332-7466-467b-8666-c09c6bc728d4-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697839
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0620fb9b-08a6-4de8-8eb8-45cb7b84f75a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0620fb9b-08a6-4de8-8eb8-45cb7b84f75a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:19
|
13 ms
|
|
[348919]
|
|
|
697772
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 068fcde6-3ab9-4f18-a21b-af780a83202e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 068fcde6-3ab9-4f18-a21b-af780a83202e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:18
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348886.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348886.0 (TID 348886) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-751298fe-bd1c-4505-8f3c-0d04d27b36ab-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348886.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348886.0 (TID 348886) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-751298fe-bd1c-4505-8f3c-0d04d27b36ab-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697773
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 068fcde6-3ab9-4f18-a21b-af780a83202e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 068fcde6-3ab9-4f18-a21b-af780a83202e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:18
|
10 ms
|
|
[348886]
|
|
|
698318
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 06996b16-6eb4-48f6-9d34-13683876fa1c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 06996b16-6eb4-48f6-9d34-13683876fa1c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:30
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349159.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349159.0 (TID 349159) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5918b370-8ccf-4ee3-9737-986513721c35-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349159.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349159.0 (TID 349159) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5918b370-8ccf-4ee3-9737-986513721c35-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698319
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 06996b16-6eb4-48f6-9d34-13683876fa1c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 06996b16-6eb4-48f6-9d34-13683876fa1c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:30
|
12 ms
|
|
[349159]
|
|
|
697958
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 06af5edb-f5b2-4dbd-968a-f7c905285a29
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 06af5edb-f5b2-4dbd-968a-f7c905285a29
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:22
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348979.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348979.0 (TID 348979) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f871471d-c48d-4804-9c84-aa29de090fda-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348979.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348979.0 (TID 348979) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f871471d-c48d-4804-9c84-aa29de090fda-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697959
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 06af5edb-f5b2-4dbd-968a-f7c905285a29
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 06af5edb-f5b2-4dbd-968a-f7c905285a29
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:22
|
12 ms
|
|
[348979]
|
|
|
697568
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 07089a3b-cf29-42d4-a3d4-9a745354bfec
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 07089a3b-cf29-42d4-a3d4-9a745354bfec
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:13
|
29 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348784.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348784.0 (TID 348784) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5b268425-3fd7-45c5-be13-64dcc15f2cbc-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348784.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348784.0 (TID 348784) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5b268425-3fd7-45c5-be13-64dcc15f2cbc-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697569
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 07089a3b-cf29-42d4-a3d4-9a745354bfec
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 07089a3b-cf29-42d4-a3d4-9a745354bfec
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:13
|
26 ms
|
|
[348784]
|
|
|
697574
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 078f9abd-ad41-4a51-9352-7fb261767483
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 078f9abd-ad41-4a51-9352-7fb261767483
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:13
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348787.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348787.0 (TID 348787) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a0a3a509-77de-4976-a9a6-2530ce76501a-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348787.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348787.0 (TID 348787) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a0a3a509-77de-4976-a9a6-2530ce76501a-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697575
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 078f9abd-ad41-4a51-9352-7fb261767483
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 078f9abd-ad41-4a51-9352-7fb261767483
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:13
|
12 ms
|
|
[348787]
|
|
|
698258
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 08b76ffc-14c7-4eb0-853d-063481a4a8b4
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 08b76ffc-14c7-4eb0-853d-063481a4a8b4
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:29
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349129.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349129.0 (TID 349129) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3d85374c-4508-4f9c-bdbe-c00e8773602c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349129.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349129.0 (TID 349129) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3d85374c-4508-4f9c-bdbe-c00e8773602c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698259
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 08b76ffc-14c7-4eb0-853d-063481a4a8b4
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 08b76ffc-14c7-4eb0-853d-063481a4a8b4
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:29
|
14 ms
|
|
[349129]
|
|
|
698416
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 09452437-608f-43d8-89e2-1d0e907b0eca
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 09452437-608f-43d8-89e2-1d0e907b0eca
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:33
|
20 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349208.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349208.0 (TID 349208) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-85b423a3-46a2-4994-9f68-eae95bfe82f5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349208.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349208.0 (TID 349208) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-85b423a3-46a2-4994-9f68-eae95bfe82f5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698417
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 09452437-608f-43d8-89e2-1d0e907b0eca
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 09452437-608f-43d8-89e2-1d0e907b0eca
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:33
|
16 ms
|
|
[349208]
|
|
|
698322
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 096a90d5-ed0e-4b2e-a31e-2b389ca48034
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 096a90d5-ed0e-4b2e-a31e-2b389ca48034
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:30
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349161.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349161.0 (TID 349161) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6a593cda-5050-4b17-96ff-174ca0d7b48e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349161.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349161.0 (TID 349161) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6a593cda-5050-4b17-96ff-174ca0d7b48e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698323
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 096a90d5-ed0e-4b2e-a31e-2b389ca48034
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 096a90d5-ed0e-4b2e-a31e-2b389ca48034
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:30
|
10 ms
|
|
[349161]
|
|
|
697670
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 097686fb-cead-435a-adc3-f04439654022
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 097686fb-cead-435a-adc3-f04439654022
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:16
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348835.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348835.0 (TID 348835) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9cd1d7f8-fcbc-4017-92e7-133fcd112742-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348835.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348835.0 (TID 348835) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-9cd1d7f8-fcbc-4017-92e7-133fcd112742-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697671
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 097686fb-cead-435a-adc3-f04439654022
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 097686fb-cead-435a-adc3-f04439654022
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:16
|
13 ms
|
|
[348835]
|
|
|
697530
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 09ff8740-0506-4485-b21a-583ccaf87b2d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 09ff8740-0506-4485-b21a-583ccaf87b2d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:12
|
20 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348765.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348765.0 (TID 348765) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-850e0a02-aeeb-45e7-9070-6e0663a8f8ba-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348765.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348765.0 (TID 348765) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-850e0a02-aeeb-45e7-9070-6e0663a8f8ba-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697531
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 09ff8740-0506-4485-b21a-583ccaf87b2d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 09ff8740-0506-4485-b21a-583ccaf87b2d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:12
|
17 ms
|
|
[348765]
|
|
|
698360
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0b24c33f-95d0-4d16-8308-b6551c06a3b2
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0b24c33f-95d0-4d16-8308-b6551c06a3b2
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:31
|
28 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349180.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349180.0 (TID 349180) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8cf5c320-b394-416d-845e-387a494b7796-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349180.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349180.0 (TID 349180) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8cf5c320-b394-416d-845e-387a494b7796-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698361
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0b24c33f-95d0-4d16-8308-b6551c06a3b2
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0b24c33f-95d0-4d16-8308-b6551c06a3b2
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:31
|
22 ms
|
|
[349180]
|
|
|
698102
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0b501ab1-a580-483f-a4d1-68795f22860e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0b501ab1-a580-483f-a4d1-68795f22860e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:25
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349051.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349051.0 (TID 349051) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-4a013418-8697-49be-83b5-e134724e6204-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349051.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349051.0 (TID 349051) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-4a013418-8697-49be-83b5-e134724e6204-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698103
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0b501ab1-a580-483f-a4d1-68795f22860e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0b501ab1-a580-483f-a4d1-68795f22860e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:25
|
13 ms
|
|
[349051]
|
|
|
698132
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0bf3c785-7349-4ce0-b4d3-477d2c76e0c9
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0bf3c785-7349-4ce0-b4d3-477d2c76e0c9
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:26
|
23 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349066.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349066.0 (TID 349066) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b9159c70-2a30-40e3-af0d-d22b2b4cbf1c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349066.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349066.0 (TID 349066) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b9159c70-2a30-40e3-af0d-d22b2b4cbf1c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698133
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0bf3c785-7349-4ce0-b4d3-477d2c76e0c9
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0bf3c785-7349-4ce0-b4d3-477d2c76e0c9
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:26
|
12 ms
|
|
[349066]
|
|
|
698396
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c4484a9-51ad-4a24-9841-5848b019e322
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c4484a9-51ad-4a24-9841-5848b019e322
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:32
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349198.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349198.0 (TID 349198) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-deeb88ba-fff7-4879-b8e7-f3ccab64517d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349198.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349198.0 (TID 349198) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-deeb88ba-fff7-4879-b8e7-f3ccab64517d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698397
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c4484a9-51ad-4a24-9841-5848b019e322
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c4484a9-51ad-4a24-9841-5848b019e322
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:32
|
14 ms
|
|
[349198]
|
|
|
698178
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c855304-2cf3-4dde-b905-513b90aa056e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c855304-2cf3-4dde-b905-513b90aa056e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:27
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349089.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349089.0 (TID 349089) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-62bc8b43-574d-4493-80cb-6ba3f9b09339-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349089.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349089.0 (TID 349089) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-62bc8b43-574d-4493-80cb-6ba3f9b09339-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698179
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c855304-2cf3-4dde-b905-513b90aa056e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0c855304-2cf3-4dde-b905-513b90aa056e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:27
|
13 ms
|
|
[349089]
|
|
|
697896
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0cdd7fcb-887e-4704-a7c1-fef343c998b6
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0cdd7fcb-887e-4704-a7c1-fef343c998b6
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:21
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348948.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348948.0 (TID 348948) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ce14aca4-6843-46d5-a3ed-9a904fd8d6cb-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348948.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348948.0 (TID 348948) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ce14aca4-6843-46d5-a3ed-9a904fd8d6cb-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697897
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0cdd7fcb-887e-4704-a7c1-fef343c998b6
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0cdd7fcb-887e-4704-a7c1-fef343c998b6
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:21
|
13 ms
|
|
[348948]
|
|
|
698172
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0d008cd7-716d-4508-8f83-43266c383bbc
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0d008cd7-716d-4508-8f83-43266c383bbc
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:27
|
18 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349086.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349086.0 (TID 349086) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-071b8391-35a0-4431-8b77-232fea3f921a-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349086.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349086.0 (TID 349086) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-071b8391-35a0-4431-8b77-232fea3f921a-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698173
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0d008cd7-716d-4508-8f83-43266c383bbc
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0d008cd7-716d-4508-8f83-43266c383bbc
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:27
|
15 ms
|
|
[349086]
|
|
|
697986
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0ed9f30f-8552-417e-8327-fd584b3abb82
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0ed9f30f-8552-417e-8327-fd584b3abb82
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:23
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348993.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348993.0 (TID 348993) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-fd905069-6155-4415-96d4-eb0d3738f166-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348993.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348993.0 (TID 348993) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-fd905069-6155-4415-96d4-eb0d3738f166-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697987
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0ed9f30f-8552-417e-8327-fd584b3abb82
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0ed9f30f-8552-417e-8327-fd584b3abb82
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:23
|
14 ms
|
|
[348993]
|
|
|
698340
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0f36b31b-c1be-46df-9047-ead9d38e14a5
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0f36b31b-c1be-46df-9047-ead9d38e14a5
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:31
|
23 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349170.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349170.0 (TID 349170) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-904e5fd6-2633-47a2-8dc7-0d9a294da9e8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349170.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349170.0 (TID 349170) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-904e5fd6-2633-47a2-8dc7-0d9a294da9e8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698341
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0f36b31b-c1be-46df-9047-ead9d38e14a5
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0f36b31b-c1be-46df-9047-ead9d38e14a5
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:31
|
19 ms
|
|
[349170]
|
|
|
698046
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0fb71aa3-f2f3-413c-acf2-e4b9d908d90c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0fb71aa3-f2f3-413c-acf2-e4b9d908d90c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:24
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349023.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349023.0 (TID 349023) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-51488944-e230-4b50-a4c9-02273547bb42-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349023.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349023.0 (TID 349023) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-51488944-e230-4b50-a4c9-02273547bb42-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698047
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0fb71aa3-f2f3-413c-acf2-e4b9d908d90c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 0fb71aa3-f2f3-413c-acf2-e4b9d908d90c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:24
|
11 ms
|
|
[349023]
|
|
|
697758
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 10fc1bcf-9bf9-4437-86ca-876b2c6ce684
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 10fc1bcf-9bf9-4437-86ca-876b2c6ce684
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:17
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348879.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348879.0 (TID 348879) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b6512765-f5b1-4450-8db8-e9b03fa93824-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348879.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348879.0 (TID 348879) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b6512765-f5b1-4450-8db8-e9b03fa93824-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697759
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 10fc1bcf-9bf9-4437-86ca-876b2c6ce684
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 10fc1bcf-9bf9-4437-86ca-876b2c6ce684
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:18
|
12 ms
|
|
[348879]
|
|
|
697974
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 11391648-e8c5-4748-8043-43833f03f1f5
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 11391648-e8c5-4748-8043-43833f03f1f5
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:22
|
19 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348987.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348987.0 (TID 348987) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c173a343-a0c1-4fbc-b5d3-5ce8fb028969-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348987.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348987.0 (TID 348987) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c173a343-a0c1-4fbc-b5d3-5ce8fb028969-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697975
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 11391648-e8c5-4748-8043-43833f03f1f5
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 11391648-e8c5-4748-8043-43833f03f1f5
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:22
|
16 ms
|
|
[348987]
|
|
|
697768
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 11ad38fd-9364-472a-8eea-6e75f81f396e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 11ad38fd-9364-472a-8eea-6e75f81f396e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:18
|
27 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348884.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348884.0 (TID 348884) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-50197146-a2c3-4727-8164-922797c3bd62-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348884.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348884.0 (TID 348884) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-50197146-a2c3-4727-8164-922797c3bd62-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697769
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 11ad38fd-9364-472a-8eea-6e75f81f396e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 11ad38fd-9364-472a-8eea-6e75f81f396e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:18
|
24 ms
|
|
[348884]
|
|
|
697616
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 11c9a03e-b591-4b9a-9d71-c17503d9cb95
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 11c9a03e-b591-4b9a-9d71-c17503d9cb95
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:14
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348808.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348808.0 (TID 348808) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8330b8de-8e18-4ccb-91ae-2090370889aa-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348808.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348808.0 (TID 348808) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8330b8de-8e18-4ccb-91ae-2090370889aa-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697617
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 11c9a03e-b591-4b9a-9d71-c17503d9cb95
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 11c9a03e-b591-4b9a-9d71-c17503d9cb95
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:14
|
13 ms
|
|
[348808]
|
|
|
698310
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1202767d-6932-4719-835c-2c1b72ac83cb
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1202767d-6932-4719-835c-2c1b72ac83cb
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:30
|
26 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349155.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349155.0 (TID 349155) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-e7fa3d89-fb91-4af3-a3df-fb949fa28e38-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349155.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349155.0 (TID 349155) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-e7fa3d89-fb91-4af3-a3df-fb949fa28e38-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698311
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1202767d-6932-4719-835c-2c1b72ac83cb
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1202767d-6932-4719-835c-2c1b72ac83cb
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:30
|
23 ms
|
|
[349155]
|
|
|
697842
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 120bf62c-9a18-44dd-8c55-a0e85294bb25
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 120bf62c-9a18-44dd-8c55-a0e85294bb25
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:19
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348921.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348921.0 (TID 348921) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a4b89c1d-5964-4790-a2ad-64cc87501f68-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348921.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348921.0 (TID 348921) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a4b89c1d-5964-4790-a2ad-64cc87501f68-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697843
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 120bf62c-9a18-44dd-8c55-a0e85294bb25
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 120bf62c-9a18-44dd-8c55-a0e85294bb25
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:19
|
12 ms
|
|
[348921]
|
|
|
697810
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 12506f09-2324-4a62-aa97-dba3945b1944
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 12506f09-2324-4a62-aa97-dba3945b1944
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:19
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348905.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348905.0 (TID 348905) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c40adca7-7d3c-44a9-a1f5-89c3eb2be923-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348905.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348905.0 (TID 348905) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-c40adca7-7d3c-44a9-a1f5-89c3eb2be923-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697811
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 12506f09-2324-4a62-aa97-dba3945b1944
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 12506f09-2324-4a62-aa97-dba3945b1944
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:19
|
12 ms
|
|
[348905]
|
|
|
698110
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 12ed4623-606f-4528-9f9e-ddbb132e7075
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 12ed4623-606f-4528-9f9e-ddbb132e7075
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:25
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349055.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349055.0 (TID 349055) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3b8a7c39-6d54-4a31-a10e-49b3efc316ae-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349055.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349055.0 (TID 349055) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3b8a7c39-6d54-4a31-a10e-49b3efc316ae-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698111
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 12ed4623-606f-4528-9f9e-ddbb132e7075
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 12ed4623-606f-4528-9f9e-ddbb132e7075
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:25
|
12 ms
|
|
[349055]
|
|
|
698408
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 13affacc-da60-4349-baad-7f2ed11e289b
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 13affacc-da60-4349-baad-7f2ed11e289b
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:32
|
28 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349204.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349204.0 (TID 349204) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-52610172-affc-4566-9752-b43771b52fb8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349204.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349204.0 (TID 349204) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-52610172-affc-4566-9752-b43771b52fb8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698409
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 13affacc-da60-4349-baad-7f2ed11e289b
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 13affacc-da60-4349-baad-7f2ed11e289b
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:32
|
24 ms
|
|
[349204]
|
|
|
698094
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 13f73e2c-5a4d-40c9-939b-27443e31f783
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 13f73e2c-5a4d-40c9-939b-27443e31f783
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:25
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349047.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349047.0 (TID 349047) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5c1a6486-a48c-4588-92db-74ee4d28c37e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349047.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349047.0 (TID 349047) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5c1a6486-a48c-4588-92db-74ee4d28c37e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698095
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 13f73e2c-5a4d-40c9-939b-27443e31f783
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 13f73e2c-5a4d-40c9-939b-27443e31f783
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:25
|
14 ms
|
|
[349047]
|
|
|
697848
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 13fa1520-ccf2-4d90-9892-45eab62c99b8
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 13fa1520-ccf2-4d90-9892-45eab62c99b8
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:19
|
35 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348924.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348924.0 (TID 348924) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a34ec1e8-657b-41f1-b5ab-6c361b7aa29d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348924.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348924.0 (TID 348924) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a34ec1e8-657b-41f1-b5ab-6c361b7aa29d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697849
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 13fa1520-ccf2-4d90-9892-45eab62c99b8
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 13fa1520-ccf2-4d90-9892-45eab62c99b8
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:19
|
32 ms
|
|
[348924]
|
|
|
697886
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 143de9f2-3c42-4ef0-b19d-96f48f0262d2
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 143de9f2-3c42-4ef0-b19d-96f48f0262d2
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:20
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348943.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348943.0 (TID 348943) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-bede4811-f891-42c6-95e3-8d552b89aeb8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348943.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348943.0 (TID 348943) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-bede4811-f891-42c6-95e3-8d552b89aeb8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697887
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 143de9f2-3c42-4ef0-b19d-96f48f0262d2
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 143de9f2-3c42-4ef0-b19d-96f48f0262d2
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:20
|
13 ms
|
|
[348943]
|
|
|
697890
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 145c68f5-ee8f-4856-ae26-5bd27ab3b2bf
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 145c68f5-ee8f-4856-ae26-5bd27ab3b2bf
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:20
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348945.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348945.0 (TID 348945) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5e465c49-68d9-46ec-9f2c-60111922de36-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348945.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348945.0 (TID 348945) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5e465c49-68d9-46ec-9f2c-60111922de36-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697891
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 145c68f5-ee8f-4856-ae26-5bd27ab3b2bf
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 145c68f5-ee8f-4856-ae26-5bd27ab3b2bf
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:20
|
14 ms
|
|
[348945]
|
|
|
697924
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15058088-b9e5-4cb1-85e4-9106adda3f59
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15058088-b9e5-4cb1-85e4-9106adda3f59
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:21
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348962.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348962.0 (TID 348962) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3e3226a6-421d-4dbc-abe7-2b537db1a689-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348962.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348962.0 (TID 348962) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-3e3226a6-421d-4dbc-abe7-2b537db1a689-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697925
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15058088-b9e5-4cb1-85e4-9106adda3f59
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15058088-b9e5-4cb1-85e4-9106adda3f59
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:21
|
14 ms
|
|
[348962]
|
|
|
698208
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 151eb1df-3c6f-4f49-9656-e5cff640d842
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 151eb1df-3c6f-4f49-9656-e5cff640d842
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:28
|
24 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349104.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349104.0 (TID 349104) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ae86766a-de6e-4183-a0f7-853c29f5458d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349104.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349104.0 (TID 349104) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ae86766a-de6e-4183-a0f7-853c29f5458d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698209
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 151eb1df-3c6f-4f49-9656-e5cff640d842
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 151eb1df-3c6f-4f49-9656-e5cff640d842
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:28
|
22 ms
|
|
[349104]
|
|
|
698168
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15351f45-f4d1-459c-a90e-f4323bb52a3e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15351f45-f4d1-459c-a90e-f4323bb52a3e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:27
|
33 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349084.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349084.0 (TID 349084) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1ec7c855-c099-4c8a-a698-a88891916fc5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349084.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349084.0 (TID 349084) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1ec7c855-c099-4c8a-a698-a88891916fc5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698169
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15351f45-f4d1-459c-a90e-f4323bb52a3e
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15351f45-f4d1-459c-a90e-f4323bb52a3e
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:27
|
29 ms
|
|
[349084]
|
|
|
697914
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15c8221f-9db5-4d1b-99d9-3137d287d29a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15c8221f-9db5-4d1b-99d9-3137d287d29a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:21
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348957.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348957.0 (TID 348957) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-cc10783d-8714-4e17-9527-585b989e6c6b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348957.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348957.0 (TID 348957) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-cc10783d-8714-4e17-9527-585b989e6c6b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697915
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15c8221f-9db5-4d1b-99d9-3137d287d29a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 15c8221f-9db5-4d1b-99d9-3137d287d29a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:21
|
12 ms
|
|
[348957]
|
|
|
697844
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 164a241a-1243-4826-9bd4-57333f861c42
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 164a241a-1243-4826-9bd4-57333f861c42
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:19
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348922.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348922.0 (TID 348922) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a00cdd86-ee14-4e56-9be4-60063605952b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348922.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348922.0 (TID 348922) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a00cdd86-ee14-4e56-9be4-60063605952b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697845
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 164a241a-1243-4826-9bd4-57333f861c42
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 164a241a-1243-4826-9bd4-57333f861c42
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:19
|
13 ms
|
|
[348922]
|
|
|
698484
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 169c189b-d945-43e9-ac7d-442612afc49d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 169c189b-d945-43e9-ac7d-442612afc49d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:34
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349242.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349242.0 (TID 349242) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ed82da65-709b-4e0a-86b4-39e67eab129b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349242.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349242.0 (TID 349242) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ed82da65-709b-4e0a-86b4-39e67eab129b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698485
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 169c189b-d945-43e9-ac7d-442612afc49d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 169c189b-d945-43e9-ac7d-442612afc49d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:34
|
14 ms
|
|
[349242]
|
|
|
697858
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 18357cbd-fa64-4a49-ac70-34a9e406b9f3
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 18357cbd-fa64-4a49-ac70-34a9e406b9f3
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:20
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348929.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348929.0 (TID 348929) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6abf1b57-f76e-4a81-bc6d-2d1400843caf-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348929.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348929.0 (TID 348929) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6abf1b57-f76e-4a81-bc6d-2d1400843caf-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697859
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 18357cbd-fa64-4a49-ac70-34a9e406b9f3
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 18357cbd-fa64-4a49-ac70-34a9e406b9f3
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:20
|
10 ms
|
|
[348929]
|
|
|
698088
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 18578a3e-06c6-464f-90b2-a6cf24cd40c3
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 18578a3e-06c6-464f-90b2-a6cf24cd40c3
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:25
|
31 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349044.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349044.0 (TID 349044) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-068b1ec5-b6a2-4ba8-86f1-62a6183fd35d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349044.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349044.0 (TID 349044) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-068b1ec5-b6a2-4ba8-86f1-62a6183fd35d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698089
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 18578a3e-06c6-464f-90b2-a6cf24cd40c3
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 18578a3e-06c6-464f-90b2-a6cf24cd40c3
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:25
|
27 ms
|
|
[349044]
|
|
|
698496
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 186f84be-64dd-4a1f-a337-9e02662b5d95
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 186f84be-64dd-4a1f-a337-9e02662b5d95
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:35
|
18 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349248.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349248.0 (TID 349248) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f9160060-dd48-4057-a0f6-66cefa43d78a-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349248.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349248.0 (TID 349248) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f9160060-dd48-4057-a0f6-66cefa43d78a-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698497
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 186f84be-64dd-4a1f-a337-9e02662b5d95
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 186f84be-64dd-4a1f-a337-9e02662b5d95
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:35
|
13 ms
|
|
[349248]
|
|
|
697916
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 188db03c-4e9d-4f56-a2a7-0004ad621c19
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 188db03c-4e9d-4f56-a2a7-0004ad621c19
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:21
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348958.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348958.0 (TID 348958) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-709f4ad8-9665-418a-80a1-3fd9acea9fb3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348958.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348958.0 (TID 348958) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-709f4ad8-9665-418a-80a1-3fd9acea9fb3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697917
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 188db03c-4e9d-4f56-a2a7-0004ad621c19
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 188db03c-4e9d-4f56-a2a7-0004ad621c19
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:21
|
12 ms
|
|
[348958]
|
|
|
698198
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 19961590-8b95-4d5b-85bf-74f31112efbf
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 19961590-8b95-4d5b-85bf-74f31112efbf
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:27
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349099.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349099.0 (TID 349099) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a41e09d6-ce53-4e4c-b343-40ff8bbdd39e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349099.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349099.0 (TID 349099) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a41e09d6-ce53-4e4c-b343-40ff8bbdd39e-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698199
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 19961590-8b95-4d5b-85bf-74f31112efbf
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 19961590-8b95-4d5b-85bf-74f31112efbf
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:27
|
13 ms
|
|
[349099]
|
|
|
697864
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 19bcb5fd-f2aa-4edb-ad0d-f615b2377cc0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 19bcb5fd-f2aa-4edb-ad0d-f615b2377cc0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:20
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348932.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348932.0 (TID 348932) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1c630609-9519-403b-a0a9-991d480e58ff-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348932.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348932.0 (TID 348932) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1c630609-9519-403b-a0a9-991d480e58ff-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697865
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 19bcb5fd-f2aa-4edb-ad0d-f615b2377cc0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 19bcb5fd-f2aa-4edb-ad0d-f615b2377cc0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:20
|
11 ms
|
|
[348932]
|
|
|
697696
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1aad6a10-163b-45c8-86aa-27e43c287f91
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1aad6a10-163b-45c8-86aa-27e43c287f91
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:16
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348848.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348848.0 (TID 348848) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8ec96738-e386-417a-85f8-f8b8278529bc-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348848.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348848.0 (TID 348848) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8ec96738-e386-417a-85f8-f8b8278529bc-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697697
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1aad6a10-163b-45c8-86aa-27e43c287f91
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1aad6a10-163b-45c8-86aa-27e43c287f91
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:16
|
11 ms
|
|
[348848]
|
|
|
698044
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1b62a264-edfe-4d42-9c09-8e9987be4078
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1b62a264-edfe-4d42-9c09-8e9987be4078
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:24
|
23 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349022.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349022.0 (TID 349022) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a9b415fe-0e40-4561-aa87-79ee760bb775-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349022.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349022.0 (TID 349022) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a9b415fe-0e40-4561-aa87-79ee760bb775-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698045
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1b62a264-edfe-4d42-9c09-8e9987be4078
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1b62a264-edfe-4d42-9c09-8e9987be4078
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:24
|
20 ms
|
|
[349022]
|
|
|
698196
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1b76a447-2631-43a2-ba05-a890c0655505
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1b76a447-2631-43a2-ba05-a890c0655505
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:27
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349098.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349098.0 (TID 349098) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-cc41c35d-eacf-492c-9f87-256ecfe9e2e5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349098.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349098.0 (TID 349098) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-cc41c35d-eacf-492c-9f87-256ecfe9e2e5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698197
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1b76a447-2631-43a2-ba05-a890c0655505
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1b76a447-2631-43a2-ba05-a890c0655505
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:27
|
12 ms
|
|
[349098]
|
|
|
698152
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1c41a6d2-549e-4de8-ab97-207917088056
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1c41a6d2-549e-4de8-ab97-207917088056
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:26
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349076.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349076.0 (TID 349076) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d55f9430-d4f9-43ae-8c1f-590a459c4bbb-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349076.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349076.0 (TID 349076) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d55f9430-d4f9-43ae-8c1f-590a459c4bbb-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698153
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1c41a6d2-549e-4de8-ab97-207917088056
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1c41a6d2-549e-4de8-ab97-207917088056
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:26
|
13 ms
|
|
[349076]
|
|
|
698124
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1d0dd401-666d-4714-9918-56592ef78796
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1d0dd401-666d-4714-9918-56592ef78796
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:26
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349062.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349062.0 (TID 349062) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2372dec6-2e55-4b58-9657-a92f5431624d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349062.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349062.0 (TID 349062) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2372dec6-2e55-4b58-9657-a92f5431624d-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698125
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1d0dd401-666d-4714-9918-56592ef78796
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1d0dd401-666d-4714-9918-56592ef78796
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:26
|
11 ms
|
|
[349062]
|
|
|
698084
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1d4d0940-61ff-4112-9776-8f544f8a0325
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1d4d0940-61ff-4112-9776-8f544f8a0325
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:25
|
20 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349042.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349042.0 (TID 349042) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-4610e485-7867-469a-a8b8-43fa532e9c27-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349042.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349042.0 (TID 349042) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-4610e485-7867-469a-a8b8-43fa532e9c27-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698085
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1d4d0940-61ff-4112-9776-8f544f8a0325
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1d4d0940-61ff-4112-9776-8f544f8a0325
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:25
|
17 ms
|
|
[349042]
|
|
|
697774
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1d9561b3-fb06-4a4b-b21a-7dfb7c6624fc
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1d9561b3-fb06-4a4b-b21a-7dfb7c6624fc
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:18
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348887.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348887.0 (TID 348887) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b7eeeb11-4ff6-4f23-bb74-1752fd8f22ee-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348887.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348887.0 (TID 348887) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b7eeeb11-4ff6-4f23-bb74-1752fd8f22ee-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697775
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1d9561b3-fb06-4a4b-b21a-7dfb7c6624fc
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1d9561b3-fb06-4a4b-b21a-7dfb7c6624fc
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:18
|
11 ms
|
|
[348887]
|
|
|
698286
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1e4266f6-1b12-4bcc-8d15-973cc9274360
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1e4266f6-1b12-4bcc-8d15-973cc9274360
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:29
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349143.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349143.0 (TID 349143) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-0e27ef0c-744e-4a9e-98cf-f0d8eafc9779-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349143.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349143.0 (TID 349143) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-0e27ef0c-744e-4a9e-98cf-f0d8eafc9779-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698287
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1e4266f6-1b12-4bcc-8d15-973cc9274360
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1e4266f6-1b12-4bcc-8d15-973cc9274360
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:29
|
12 ms
|
|
[349143]
|
|
|
698350
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1ee2caa1-0934-4691-8338-c947b93f4b48
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1ee2caa1-0934-4691-8338-c947b93f4b48
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:31
|
20 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349175.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349175.0 (TID 349175) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-caecd403-0b15-4008-91b3-35094c18ecd2-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349175.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349175.0 (TID 349175) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-caecd403-0b15-4008-91b3-35094c18ecd2-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698351
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1ee2caa1-0934-4691-8338-c947b93f4b48
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1ee2caa1-0934-4691-8338-c947b93f4b48
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:31
|
16 ms
|
|
[349175]
|
|
|
698026
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1f67e445-b834-4a8f-aa84-9928d8c10abe
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1f67e445-b834-4a8f-aa84-9928d8c10abe
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:23
|
23 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349013.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349013.0 (TID 349013) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d9b5924f-7194-4394-b7dd-bb04624ee56b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349013.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349013.0 (TID 349013) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d9b5924f-7194-4394-b7dd-bb04624ee56b-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698027
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1f67e445-b834-4a8f-aa84-9928d8c10abe
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1f67e445-b834-4a8f-aa84-9928d8c10abe
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:23
|
20 ms
|
|
[349013]
|
|
|
697522
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1f99d5ad-c62b-438c-93e8-5cb4ff2abf13
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1f99d5ad-c62b-438c-93e8-5cb4ff2abf13
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:12
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348761.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348761.0 (TID 348761) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-47fdffec-eff1-42ca-8538-fe02a7fd6782-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348761.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348761.0 (TID 348761) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-47fdffec-eff1-42ca-8538-fe02a7fd6782-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697523
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1f99d5ad-c62b-438c-93e8-5cb4ff2abf13
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1f99d5ad-c62b-438c-93e8-5cb4ff2abf13
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:12
|
11 ms
|
|
[348761]
|
|
|
697846
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1fa94ced-c74e-4d18-af43-26bf0d51cfad
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1fa94ced-c74e-4d18-af43-26bf0d51cfad
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:19
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348923.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348923.0 (TID 348923) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6780c674-3004-4bf1-9b51-e2b2d9fac880-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348923.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348923.0 (TID 348923) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6780c674-3004-4bf1-9b51-e2b2d9fac880-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697847
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1fa94ced-c74e-4d18-af43-26bf0d51cfad
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 1fa94ced-c74e-4d18-af43-26bf0d51cfad
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:19
|
13 ms
|
|
[348923]
|
|
|
698156
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 202a1ab5-eb42-4bd1-8e26-ca418ba2169d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 202a1ab5-eb42-4bd1-8e26-ca418ba2169d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:26
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349078.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349078.0 (TID 349078) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-657f4189-7d02-4f74-9a27-20072768c191-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349078.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349078.0 (TID 349078) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-657f4189-7d02-4f74-9a27-20072768c191-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698157
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 202a1ab5-eb42-4bd1-8e26-ca418ba2169d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 202a1ab5-eb42-4bd1-8e26-ca418ba2169d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:26
|
13 ms
|
|
[349078]
|
|
|
697600
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 209b4e7c-88db-48ce-90b6-7a9f1a110bac
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 209b4e7c-88db-48ce-90b6-7a9f1a110bac
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:14
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348800.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348800.0 (TID 348800) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-cea3fc84-cdcb-4e10-8099-3bab349ccd39-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348800.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348800.0 (TID 348800) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-cea3fc84-cdcb-4e10-8099-3bab349ccd39-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697601
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 209b4e7c-88db-48ce-90b6-7a9f1a110bac
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 209b4e7c-88db-48ce-90b6-7a9f1a110bac
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:14
|
11 ms
|
|
[348800]
|
|
|
698382
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 228a526d-773a-4f5f-8063-562b296832c7
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 228a526d-773a-4f5f-8063-562b296832c7
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:32
|
32 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349191.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349191.0 (TID 349191) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6f2e70d4-7210-449e-aa88-3b565dbde7db-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349191.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349191.0 (TID 349191) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6f2e70d4-7210-449e-aa88-3b565dbde7db-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698383
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 228a526d-773a-4f5f-8063-562b296832c7
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 228a526d-773a-4f5f-8063-562b296832c7
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:32
|
27 ms
|
|
[349191]
|
|
|
697732
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 229bdce3-a424-4d20-b35a-3ac6d1b9a8fa
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 229bdce3-a424-4d20-b35a-3ac6d1b9a8fa
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:17
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348866.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348866.0 (TID 348866) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-32bb8b3b-087f-493e-afea-4e56d9d76b77-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348866.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348866.0 (TID 348866) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-32bb8b3b-087f-493e-afea-4e56d9d76b77-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697733
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 229bdce3-a424-4d20-b35a-3ac6d1b9a8fa
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 229bdce3-a424-4d20-b35a-3ac6d1b9a8fa
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:17
|
12 ms
|
|
[348866]
|
|
|
697994
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 240b9fd6-4f84-4b96-b82d-f498a53d30dc
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 240b9fd6-4f84-4b96-b82d-f498a53d30dc
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:23
|
16 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348997.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348997.0 (TID 348997) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ddd6562e-d0b8-4578-8023-60c4512ace3f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348997.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348997.0 (TID 348997) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-ddd6562e-d0b8-4578-8023-60c4512ace3f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697995
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 240b9fd6-4f84-4b96-b82d-f498a53d30dc
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 240b9fd6-4f84-4b96-b82d-f498a53d30dc
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:23
|
13 ms
|
|
[348997]
|
|
|
698336
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 25c37405-1564-4a88-a3e6-ad06f2601bfe
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 25c37405-1564-4a88-a3e6-ad06f2601bfe
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:30
|
21 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349168.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349168.0 (TID 349168) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-76d0a498-07a3-463b-8019-ba0a86d3a96c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349168.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349168.0 (TID 349168) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-76d0a498-07a3-463b-8019-ba0a86d3a96c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698337
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 25c37405-1564-4a88-a3e6-ad06f2601bfe
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 25c37405-1564-4a88-a3e6-ad06f2601bfe
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:30
|
17 ms
|
|
[349168]
|
|
|
697578
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 26c0cf80-0449-4fb6-b3ee-f390831da63a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 26c0cf80-0449-4fb6-b3ee-f390831da63a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:13
|
29 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348789.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348789.0 (TID 348789) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-148eaa9b-c92c-4dfd-8f42-b8ebcfbdcaeb-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348789.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348789.0 (TID 348789) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-148eaa9b-c92c-4dfd-8f42-b8ebcfbdcaeb-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697579
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 26c0cf80-0449-4fb6-b3ee-f390831da63a
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 26c0cf80-0449-4fb6-b3ee-f390831da63a
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:13
|
26 ms
|
|
[348789]
|
|
|
698324
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 26e89ca2-d725-467d-bb7e-89772fea06ec
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 26e89ca2-d725-467d-bb7e-89772fea06ec
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:30
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349162.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349162.0 (TID 349162) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-42fa4ef4-844b-4299-aaaa-b04da04de470-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349162.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349162.0 (TID 349162) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-42fa4ef4-844b-4299-aaaa-b04da04de470-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698325
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 26e89ca2-d725-467d-bb7e-89772fea06ec
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 26e89ca2-d725-467d-bb7e-89772fea06ec
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:30
|
11 ms
|
|
[349162]
|
|
|
698492
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 27746548-35bd-4b80-8987-76ecc2b7480d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 27746548-35bd-4b80-8987-76ecc2b7480d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:35
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349246.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349246.0 (TID 349246) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-64bac7d5-f265-4dba-98a8-1129aadaf1c2-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349246.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349246.0 (TID 349246) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-64bac7d5-f265-4dba-98a8-1129aadaf1c2-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698493
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 27746548-35bd-4b80-8987-76ecc2b7480d
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 27746548-35bd-4b80-8987-76ecc2b7480d
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:35
|
13 ms
|
|
[349246]
|
|
|
698136
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 27d594e4-6848-4589-9ac0-fb3a400d0aac
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 27d594e4-6848-4589-9ac0-fb3a400d0aac
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:26
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349068.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349068.0 (TID 349068) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-35dd13c7-0cee-4cf0-822e-1b799bf45bf6-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349068.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349068.0 (TID 349068) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-35dd13c7-0cee-4cf0-822e-1b799bf45bf6-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698137
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 27d594e4-6848-4589-9ac0-fb3a400d0aac
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 27d594e4-6848-4589-9ac0-fb3a400d0aac
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:26
|
11 ms
|
|
[349068]
|
|
|
697532
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 29cacbd3-e369-4c25-ac73-a4e3708c561f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 29cacbd3-e369-4c25-ac73-a4e3708c561f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:12
|
24 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348766.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348766.0 (TID 348766) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2f78c34b-183a-4ebe-9add-df2123b396c3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348766.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348766.0 (TID 348766) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-2f78c34b-183a-4ebe-9add-df2123b396c3-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697533
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 29cacbd3-e369-4c25-ac73-a4e3708c561f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 29cacbd3-e369-4c25-ac73-a4e3708c561f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:12
|
21 ms
|
|
[348766]
|
|
|
697746
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2a4d18f3-3d10-4fcd-b883-a0dc21ce1494
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2a4d18f3-3d10-4fcd-b883-a0dc21ce1494
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:17
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348873.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348873.0 (TID 348873) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-962c791e-7e07-434d-bf29-c10d1bb780cd-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348873.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348873.0 (TID 348873) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-962c791e-7e07-434d-bf29-c10d1bb780cd-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697747
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2a4d18f3-3d10-4fcd-b883-a0dc21ce1494
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2a4d18f3-3d10-4fcd-b883-a0dc21ce1494
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:17
|
12 ms
|
|
[348873]
|
|
|
698346
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2a7e94b0-c040-43ce-861f-db4f61f4d145
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2a7e94b0-c040-43ce-861f-db4f61f4d145
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:31
|
24 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349173.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349173.0 (TID 349173) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-0d2472f7-6f36-41c7-b027-b705da96da04-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349173.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349173.0 (TID 349173) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-0d2472f7-6f36-41c7-b027-b705da96da04-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698347
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2a7e94b0-c040-43ce-861f-db4f61f4d145
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2a7e94b0-c040-43ce-861f-db4f61f4d145
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:31
|
18 ms
|
|
[349173]
|
|
|
698390
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2aaf3cea-2aa0-4dfc-b614-49664ed531dd
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2aaf3cea-2aa0-4dfc-b614-49664ed531dd
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:32
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349195.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349195.0 (TID 349195) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d195b400-b611-42b6-b8c5-58516da0e28f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349195.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349195.0 (TID 349195) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-d195b400-b611-42b6-b8c5-58516da0e28f-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698391
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2aaf3cea-2aa0-4dfc-b614-49664ed531dd
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2aaf3cea-2aa0-4dfc-b614-49664ed531dd
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:32
|
14 ms
|
|
[349195]
|
|
|
698488
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2aee22dd-662d-4c07-ba2c-cf466b32c680
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2aee22dd-662d-4c07-ba2c-cf466b32c680
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:35
|
29 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349244.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349244.0 (TID 349244) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f8467ba1-aaf4-4401-a3dc-6f592b46b294-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349244.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349244.0 (TID 349244) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-f8467ba1-aaf4-4401-a3dc-6f592b46b294-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698489
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2aee22dd-662d-4c07-ba2c-cf466b32c680
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2aee22dd-662d-4c07-ba2c-cf466b32c680
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:35
|
25 ms
|
|
[349244]
|
|
|
697794
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2d3b503d-f363-4676-8bc0-3310da728a1c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2d3b503d-f363-4676-8bc0-3310da728a1c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:18
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348897.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348897.0 (TID 348897) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-e2737c65-fd38-49da-aa93-e484ea04eed9-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348897.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348897.0 (TID 348897) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-e2737c65-fd38-49da-aa93-e484ea04eed9-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697795
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2d3b503d-f363-4676-8bc0-3310da728a1c
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2d3b503d-f363-4676-8bc0-3310da728a1c
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:18
|
10 ms
|
|
[348897]
|
|
|
697594
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2e9b2865-3a14-40ab-916b-85d874847838
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2e9b2865-3a14-40ab-916b-85d874847838
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:14
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348797.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348797.0 (TID 348797) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-154b0c46-7d45-41bd-b536-ef498922f7b0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348797.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348797.0 (TID 348797) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-154b0c46-7d45-41bd-b536-ef498922f7b0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697595
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2e9b2865-3a14-40ab-916b-85d874847838
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2e9b2865-3a14-40ab-916b-85d874847838
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:14
|
14 ms
|
|
[348797]
|
|
|
698332
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2eb961e8-c38b-46ae-af6b-d5eafeaea2ca
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2eb961e8-c38b-46ae-af6b-d5eafeaea2ca
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:30
|
17 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349166.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349166.0 (TID 349166) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-cf868965-0b00-40b0-8108-d9d4bdc0b221-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349166.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349166.0 (TID 349166) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-cf868965-0b00-40b0-8108-d9d4bdc0b221-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698333
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2eb961e8-c38b-46ae-af6b-d5eafeaea2ca
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2eb961e8-c38b-46ae-af6b-d5eafeaea2ca
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:30
|
15 ms
|
|
[349166]
|
|
|
698028
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2f774675-25f0-4996-8cc3-ba3deca55e66
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2f774675-25f0-4996-8cc3-ba3deca55e66
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:24
|
29 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349014.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349014.0 (TID 349014) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a0f5a40a-6546-455e-9ba0-33bacfc5cbf0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349014.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349014.0 (TID 349014) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-a0f5a40a-6546-455e-9ba0-33bacfc5cbf0-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698029
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2f774675-25f0-4996-8cc3-ba3deca55e66
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 2f774675-25f0-4996-8cc3-ba3deca55e66
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:24
|
25 ms
|
|
[349014]
|
|
|
698086
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 30454a63-402e-48bc-83e4-303b9e14eecd
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 30454a63-402e-48bc-83e4-303b9e14eecd
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:25
|
18 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349043.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349043.0 (TID 349043) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1655ca83-1b22-4466-9a4a-25932c1c50a1-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349043.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349043.0 (TID 349043) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-1655ca83-1b22-4466-9a4a-25932c1c50a1-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698087
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 30454a63-402e-48bc-83e4-303b9e14eecd
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 30454a63-402e-48bc-83e4-303b9e14eecd
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:25
|
14 ms
|
|
[349043]
|
|
|
697646
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3075048d-e69d-4842-840f-c206e23bfb5f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3075048d-e69d-4842-840f-c206e23bfb5f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:15
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348823.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348823.0 (TID 348823) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6e727177-443a-47ca-ab6e-4542898479d8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348823.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348823.0 (TID 348823) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6e727177-443a-47ca-ab6e-4542898479d8-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697647
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3075048d-e69d-4842-840f-c206e23bfb5f
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3075048d-e69d-4842-840f-c206e23bfb5f
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:15
|
13 ms
|
|
[348823]
|
|
|
697854
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 311a1479-c7c7-4618-adbf-a69e3f321975
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 311a1479-c7c7-4618-adbf-a69e3f321975
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:20
|
14 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348927.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348927.0 (TID 348927) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-16a9a25f-3cee-4753-83bd-3d1c27d9f6ff-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348927.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348927.0 (TID 348927) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-16a9a25f-3cee-4753-83bd-3d1c27d9f6ff-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697855
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 311a1479-c7c7-4618-adbf-a69e3f321975
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 311a1479-c7c7-4618-adbf-a69e3f321975
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:20
|
12 ms
|
|
[348927]
|
|
|
698260
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 312b7bcd-cc6a-4679-a175-13600bc9be45
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 312b7bcd-cc6a-4679-a175-13600bc9be45
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:29
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349130.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349130.0 (TID 349130) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6a5b76ce-f3aa-4c0c-bdc0-c34007bf9a1c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349130.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349130.0 (TID 349130) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-6a5b76ce-f3aa-4c0c-bdc0-c34007bf9a1c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698261
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 312b7bcd-cc6a-4679-a175-13600bc9be45
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 312b7bcd-cc6a-4679-a175-13600bc9be45
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:29
|
13 ms
|
|
[349130]
|
|
|
697962
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 318028bb-6e13-4ebc-a4a7-c4dc62a37802
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 318028bb-6e13-4ebc-a4a7-c4dc62a37802
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:22
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348981.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348981.0 (TID 348981) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-83648e41-b44b-4945-8d33-470eb2e049cf-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348981.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348981.0 (TID 348981) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-83648e41-b44b-4945-8d33-470eb2e049cf-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697963
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 318028bb-6e13-4ebc-a4a7-c4dc62a37802
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 318028bb-6e13-4ebc-a4a7-c4dc62a37802
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:22
|
11 ms
|
|
[348981]
|
|
|
698226
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3185f463-7e80-4084-a5f0-7ecd108b2885
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3185f463-7e80-4084-a5f0-7ecd108b2885
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:28
|
12 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349113.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349113.0 (TID 349113) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8b115a87-4831-433e-8969-6cc6921a4137-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349113.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349113.0 (TID 349113) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-8b115a87-4831-433e-8969-6cc6921a4137-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698227
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3185f463-7e80-4084-a5f0-7ecd108b2885
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3185f463-7e80-4084-a5f0-7ecd108b2885
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:28
|
10 ms
|
|
[349113]
|
|
|
697606
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 31bb5797-dca9-4f5d-9677-47ec814e1d79
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 31bb5797-dca9-4f5d-9677-47ec814e1d79
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:14
|
22 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348803.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348803.0 (TID 348803) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-7bfc6f84-9b38-4568-abc3-ab450dab398c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348803.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348803.0 (TID 348803) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-7bfc6f84-9b38-4568-abc3-ab450dab398c-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697607
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 31bb5797-dca9-4f5d-9677-47ec814e1d79
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 31bb5797-dca9-4f5d-9677-47ec814e1d79
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:14
|
20 ms
|
|
[348803]
|
|
|
697598
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3208f69a-c87b-47b8-a8d0-83817046c950
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3208f69a-c87b-47b8-a8d0-83817046c950
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:14
|
15 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348799.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348799.0 (TID 348799) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-aab20a1c-a753-4a01-93c9-8041cff6a705-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348799.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348799.0 (TID 348799) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-aab20a1c-a753-4a01-93c9-8041cff6a705-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697599
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3208f69a-c87b-47b8-a8d0-83817046c950
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 3208f69a-c87b-47b8-a8d0-83817046c950
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:14
|
13 ms
|
|
[348799]
|
|
|
697704
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 321335ba-8bcd-4d5a-aedf-07289a29c5ca
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 321335ba-8bcd-4d5a-aedf-07289a29c5ca
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:16
|
13 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 348852.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348852.0 (TID 348852) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5994acf1-9df6-4858-86fc-d1fdac922035-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 348852.0 failed 1 times, most recent failure: Lost task 0.0 in stage 348852.0 (TID 348852) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-5994acf1-9df6-4858-86fc-d1fdac922035-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
697705
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 321335ba-8bcd-4d5a-aedf-07289a29c5ca
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 321335ba-8bcd-4d5a-aedf-07289a29c5ca
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:16
|
11 ms
|
|
[348852]
|
|
|
698414
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 322557eb-00b7-4804-8609-138076ad9ff0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 322557eb-00b7-4804-8609-138076ad9ff0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:33
|
20 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349207.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349207.0 (TID 349207) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b81c1aa4-5937-4808-826a-a656f4ec8b72-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349207.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349207.0 (TID 349207) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-b81c1aa4-5937-4808-826a-a656f4ec8b72-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698415
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 322557eb-00b7-4804-8609-138076ad9ff0
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 322557eb-00b7-4804-8609-138076ad9ff0
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:33
|
16 ms
|
|
[349207]
|
|
|
698406
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 328a153d-cfe8-4879-98f5-bc8c0e609bf3
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 328a153d-cfe8-4879-98f5-bc8c0e609bf3
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:32
|
18 ms
|
|
|
Job aborted due to stage failure: Task 0 in stage 349203.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349203.0 (TID 349203) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-bf46b080-90b3-4f02-90fc-ece0e3a3e2f5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
+details
Job aborted due to stage failure: Task 0 in stage 349203.0 failed 1 times, most recent failure: Lost task 0.0 in stage 349203.0 (TID 349203) (2d47ef339462 executor driver): java.lang.IllegalStateException: Cannot fetch offset 12 (GroupId: spark-kafka-source-bf46b080-90b3-4f02-90fc-ece0e3a3e2f5-1980890413-executor, TopicPartition: mothership.masteryconnect.classroom_teachers-0).
Some data may have been lost because they are not available in Kafka any more; either the
data was aged out by Kafka or the topic may have been deleted before all the data in the
topic was processed. If you don't want your streaming query to fail on such cases, set the
source option "failOnDataLoss" to "false".
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer$.org$apache$spark$sql$kafka010$consumer$KafkaDataConsumer$$reportDataLoss0(KafkaDataConsumer.scala:724)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.reportDataLoss(KafkaDataConsumer.scala:651)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:344)
at org.apache.spark.util.UninterruptibleThread.runUninterruptibly(UninterruptibleThread.scala:149)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.runUninterruptiblyIfPossible(KafkaDataConsumer.scala:656)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.get(KafkaDataConsumer.scala:299)
at org.apache.spark.sql.kafka010.KafkaBatchPartitionReader.next(KafkaBatchPartitionReader.scala:79)
at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:146)
at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:184)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:71)
at scala.Option.exists(Option.scala:376)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:102)
at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:71)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:388)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:893)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:893)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:621)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:624)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.kafka.clients.consumer.OffsetOutOfRangeException: Fetch position FetchPosition{offset=12, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 1001 rack: null)], epoch=0}} is out of range for partition mothership.masteryconnect.classroom_teachers-0
at org.apache.kafka.clients.consumer.internals.FetchCollector.handleInitializeErrors(FetchCollector.java:365)
at org.apache.kafka.clients.consumer.internals.FetchCollector.initialize(FetchCollector.java:231)
at org.apache.kafka.clients.consumer.internals.FetchCollector.collectFetch(FetchCollector.java:111)
at org.apache.kafka.clients.consumer.internals.Fetcher.collectFetch(Fetcher.java:146)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:699)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.spark.sql.kafka010.consumer.InternalKafkaConsumer.fetch(KafkaDataConsumer.scala:78)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$fetchData$1(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.timeNanos(KafkaDataConsumer.scala:666)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchData(KafkaDataConsumer.scala:579)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.fetchRecord(KafkaDataConsumer.scala:502)
at org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.$anonfun$get$1(KafkaDataConsumer.scala:323)
... 37 more
Driver stacktrace:
|
+details
|
|
|
| ID | Description | Submitted | Duration | Succeeded Job IDs | Failed Job IDs | Error Message |
|
698407
|
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 328a153d-cfe8-4879-98f5-bc8c0e609bf3
batch = 106
+details
id = 9848c4f2-c6f1-4b94-a125-6e64ef893aaa
runId = 328a153d-cfe8-4879-98f5-bc8c0e609bf3
batch = 106
org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
data.kafka.KafkaDataFrameDataSource.monitor(KafkaDataFrameDataSource.scala:46)
data.kafka.KafkaDataFrameDataSource.start(KafkaDataFrameDataSource.scala:23)
Main$.$anonfun$main$2(Main.scala:28)
Main$.$anonfun$main$2$adapted(Main.scala:28)
scala.collection.immutable.Set$Set1.foreach(Set.scala:141)
Main$.main(Main.scala:28)
Main.main(Main.scala)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1034)
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:199)
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:222)
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1125)
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1134)
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
|
2026/09/03 13:43:32
|
15 ms
|
|
[349203]
|
|