为什么 Scala 的堆栈会因为这个 Spark API 调用而溢出？
Why Scala's stack is got overflow there for this Spark API calls?
在 Apache Zeppelin 中，很难发现这个表达式有什么问题：
val dailySummariesDf =
getDFFromJdbcSource(SparkSession.builder().appName("test").master("local").getOrCreate(),
s"SELECT * FROM mf_joined WHERE (timestamp > '2022-01-24 08:00:00.000')")
.withColumn("DiffInSeconds", to_timestamp(date_format(col("writetime"), "HH:mm:ss.SSS")).cast(LongType) -
to_timestamp(date_format(col("timestamp"), "HH:mm:ss.SSS")).cast(LongType))
.withColumn("time", to_timestamp(date_format(col("timestamp"), "HH:mm:ss.SSS")))
//.withColumn("hour",date_trunc("hour",($"unit_ts")))
.withColumn("hour",date_trunc("minute",($"writetime")))
.withColumn("max",max("DiffInSeconds").over(Window.partitionBy($"hour")))
.withColumn("min",min("DiffInSeconds").over(Window.partitionBy($"hour")))
.withColumn("mean",mean("DiffInSeconds").over(Window.partitionBy($"hour")))
.withColumn("count",count("DiffInSeconds").over(Window.partitionBy($"hour")))
.withColumn("max2",when(col("max") > 100, 101).otherwise(col("max")))
.withColumn("min2",when(col("min") < -100, -101).otherwise(col("min")))
.withColumn("mean2",when(col("mean") < -100, -101).otherwise(col("mean")))
.withColumn("rnb",row_number().over(Window.partitionBy($"hour").orderBy($"timestamp".desc)))
.where($"rnb"===1)
//.select($"hour",$"max",$"min",$"mean",$"max2",$"min2",$"mean2",$"count")
.orderBy($"hour".desc)
.persist(StorageLevel.MEMORY_ONLY_SER)
//orderBy("windowstart")
我收到错误消息
java.lang.WhosebugError
at scala.reflect.internal.Types$TypeRef.isTrivial(Types.scala:2081)
at scala.reflect.internal.Types$TypeRef.isTrivial(Types.scala:2081)
at scala.reflect.internal.Types$TypeRef.isTrivial(Types.scala:2081)
at scala.reflect.internal.Types$MethodType.isTrivialResult(Types.scala:2476)
at scala.reflect.internal.Types$MethodType.isTrivial(Types.scala:2471)
at scala.reflect.internal.Types$Type.asSeenFrom(Types.scala:698)
at scala.reflect.internal.Types$Type.computeMemberType(Types.scala:740)
at scala.reflect.internal.Symbols$MethodSymbol.typeAsMemberOf(Symbols.scala:3008)
at scala.reflect.internal.Types$Type.memberType(Types.scala:731)
at scala.tools.nsc.typechecker.Infer$Inferencer.checkAccessible(Infer.scala:282)
at scala.tools.nsc.typechecker.Typers$Typer.makeAccessible(Typers.scala:592)
at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1(Typers.scala:5081)
at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$silent(Typers.scala:727)
at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:727)
at scala.tools.nsc.typechecker.Typers$Typer.typedSelect(Typers.scala:5081)
at scala.tools.nsc.typechecker.Typers$Typer.typedSelectOrSuperCall(Typers.scala:5138)
at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5671)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1(Typers.scala:4837)
at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:714)
at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply(Typers.scala:4839)
at scala.tools.nsc.typechecker.Typers$Typer.typedApply(Typers.scala:4867)
at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5670)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.transform.Erasure$Eraser.adaptMember(Erasure.scala:734)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1(Typers.scala:4837)
at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:714)
at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply(Typers.scala:4839)
at scala.tools.nsc.typechecker.Typers$Typer.typedApply(Typers.scala:4867)
at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5670)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.transform.Erasure$Eraser.adaptMember(Erasure.scala:734)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1(Typers.scala:4837)
at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:714)
at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply(Typers.scala:4839)
at scala.tools.nsc.typechecker.Typers$Typer.typedApply(Typers.scala:4867)
at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5670)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.transform.Erasure$Eraser.adaptMember(Erasure.scala:734)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1(Typers.scala:4837)
at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:714)
at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply(Typers.scala:4839)
at scala.tools.nsc.typechecker.Typers$Typer.typedApply(Typers.scala:4867)
at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5670)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.transform.Erasure$Eraser.adaptMember(Erasure.scala:734)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1(Typers.scala:4837)
at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:714)
at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply(Typers.scala:4839)
at scala.tools.nsc.typechecker.Typers$Typer.typedApply(Typers.scala:4867)
at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5670)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.transform.Erasure$Eraser.adaptMember(Erasure.scala:734)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
对于 Apache Zeppelin，只要在 Databook 的设置中重新启动解释器就足够了，之后问题就消失了。但来历尚不清楚
在 Apache Zeppelin 中，很难发现这个表达式有什么问题：
val dailySummariesDf =
getDFFromJdbcSource(SparkSession.builder().appName("test").master("local").getOrCreate(),
s"SELECT * FROM mf_joined WHERE (timestamp > '2022-01-24 08:00:00.000')")
.withColumn("DiffInSeconds", to_timestamp(date_format(col("writetime"), "HH:mm:ss.SSS")).cast(LongType) -
to_timestamp(date_format(col("timestamp"), "HH:mm:ss.SSS")).cast(LongType))
.withColumn("time", to_timestamp(date_format(col("timestamp"), "HH:mm:ss.SSS")))
//.withColumn("hour",date_trunc("hour",($"unit_ts")))
.withColumn("hour",date_trunc("minute",($"writetime")))
.withColumn("max",max("DiffInSeconds").over(Window.partitionBy($"hour")))
.withColumn("min",min("DiffInSeconds").over(Window.partitionBy($"hour")))
.withColumn("mean",mean("DiffInSeconds").over(Window.partitionBy($"hour")))
.withColumn("count",count("DiffInSeconds").over(Window.partitionBy($"hour")))
.withColumn("max2",when(col("max") > 100, 101).otherwise(col("max")))
.withColumn("min2",when(col("min") < -100, -101).otherwise(col("min")))
.withColumn("mean2",when(col("mean") < -100, -101).otherwise(col("mean")))
.withColumn("rnb",row_number().over(Window.partitionBy($"hour").orderBy($"timestamp".desc)))
.where($"rnb"===1)
//.select($"hour",$"max",$"min",$"mean",$"max2",$"min2",$"mean2",$"count")
.orderBy($"hour".desc)
.persist(StorageLevel.MEMORY_ONLY_SER)
//orderBy("windowstart")
我收到错误消息
java.lang.WhosebugError
at scala.reflect.internal.Types$TypeRef.isTrivial(Types.scala:2081)
at scala.reflect.internal.Types$TypeRef.isTrivial(Types.scala:2081)
at scala.reflect.internal.Types$TypeRef.isTrivial(Types.scala:2081)
at scala.reflect.internal.Types$MethodType.isTrivialResult(Types.scala:2476)
at scala.reflect.internal.Types$MethodType.isTrivial(Types.scala:2471)
at scala.reflect.internal.Types$Type.asSeenFrom(Types.scala:698)
at scala.reflect.internal.Types$Type.computeMemberType(Types.scala:740)
at scala.reflect.internal.Symbols$MethodSymbol.typeAsMemberOf(Symbols.scala:3008)
at scala.reflect.internal.Types$Type.memberType(Types.scala:731)
at scala.tools.nsc.typechecker.Infer$Inferencer.checkAccessible(Infer.scala:282)
at scala.tools.nsc.typechecker.Typers$Typer.makeAccessible(Typers.scala:592)
at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1(Typers.scala:5081)
at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$silent(Typers.scala:727)
at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:727)
at scala.tools.nsc.typechecker.Typers$Typer.typedSelect(Typers.scala:5081)
at scala.tools.nsc.typechecker.Typers$Typer.typedSelectOrSuperCall(Typers.scala:5138)
at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5671)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1(Typers.scala:4837)
at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:714)
at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply(Typers.scala:4839)
at scala.tools.nsc.typechecker.Typers$Typer.typedApply(Typers.scala:4867)
at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5670)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.transform.Erasure$Eraser.adaptMember(Erasure.scala:734)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1(Typers.scala:4837)
at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:714)
at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply(Typers.scala:4839)
at scala.tools.nsc.typechecker.Typers$Typer.typedApply(Typers.scala:4867)
at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5670)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.transform.Erasure$Eraser.adaptMember(Erasure.scala:734)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1(Typers.scala:4837)
at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:714)
at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply(Typers.scala:4839)
at scala.tools.nsc.typechecker.Typers$Typer.typedApply(Typers.scala:4867)
at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5670)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.transform.Erasure$Eraser.adaptMember(Erasure.scala:734)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1(Typers.scala:4837)
at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:714)
at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply(Typers.scala:4839)
at scala.tools.nsc.typechecker.Typers$Typer.typedApply(Typers.scala:4867)
at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5670)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.transform.Erasure$Eraser.adaptMember(Erasure.scala:734)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1(Typers.scala:4837)
at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:714)
at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply(Typers.scala:4839)
at scala.tools.nsc.typechecker.Typers$Typer.typedApply(Typers.scala:4867)
at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5670)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
at scala.tools.nsc.transform.Erasure$Eraser.adaptMember(Erasure.scala:734)
at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
对于 Apache Zeppelin，只要在 Databook 的设置中重新启动解释器就足够了，之后问题就消失了。但来历尚不清楚