apache
[Bug] Excuting spark core `spark.sql("add jar path/xxxx")`,if the path is hdfs or other remote path will occur npe
Code of Conduct
- [X] I agree to follow this project's Code of Conduct
Search before asking
- [X] I have searched in the issues and found no similar issues.
Describe the bug
Excuting spark core spark.sql("add jar path/xxxx"), if the path is hdfs or other remote path will occur npe
Excuting spark core spark.sql("add jar path/xxxx") if the path is local path in driver node will be ok.
Based on issue #2471 #2475
Affects Version(s)
master
Kyuubi Server Log Output
No response
Kyuubi Engine Log Output
` 22/06/20 17:14:27 ERROR ExecuteScala: Error operating EXECUTE_STATEMENT: java.lang.NullPointerException at scala.tools.nsc.classpath.FileUtils$AbstractFileOps$.isJarOrZip$extension(FileUtils.scala:32) at scala.tools.nsc.classpath.ClassPathFactory$.newClassPath(ClassPathFactory.scala:91) at scala.tools.nsc.Global.$anonfun$extendCompilerClassPath$1(Global.scala:825) at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238) at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36) at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33) at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38) at scala.collection.TraversableLike.map(TraversableLike.scala:238) at scala.collection.TraversableLike.map$(TraversableLike.scala:231) at scala.collection.AbstractTraversable.map(Traversable.scala:108) at scala.tools.nsc.Global.extendCompilerClassPath(Global.scala:825) at scala.tools.nsc.interpreter.IMain.addUrlsToClassPath(IMain.scala:259) at org.apache.kyuubi.engine.spark.operation.ExecuteScala.$anonfun$runInternal$1(ExecuteScala.scala:69) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at org.apache.kyuubi.engine.spark.operation.SparkOperation.withLocalProperties(SparkOperation.scala:88) at org.apache.kyuubi.engine.spark.operation.ExecuteScala.runInternal(ExecuteScala.scala:60) at org.apache.kyuubi.operation.AbstractOperation.run(AbstractOperation.scala:133) at org.apache.kyuubi.session.AbstractSession.runOperation(AbstractSession.scala:94) at org.apache.kyuubi.engine.spark.session.SparkSessionImpl.runOperation(SparkSessionImpl.scala:66) at org.apache.kyuubi.session.AbstractSession.$anonfun$executeStatement$1(AbstractSession.scala:124) at org.apache.kyuubi.session.AbstractSession.withAcquireRelease(AbstractSession.scala:75) at org.apache.kyuubi.session.AbstractSession.executeStatement(AbstractSession.scala:121) at org.apache.kyuubi.service.AbstractBackendService.executeStatement(AbstractBackendService.scala:66) at org.apache.kyuubi.service.TFrontendService.ExecuteStatement(TFrontendService.scala:232) at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1557) at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1542) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:38) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.kyuubi.service.authentication.TSetIpAddressProcessor.process(TSetIpAddressProcessor.scala:36) at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:310) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) java.lang.NullPointerException at scala.tools.nsc.classpath.FileUtils$AbstractFileOps$.isJarOrZip$extension(FileUtils.scala:32) at scala.tools.nsc.classpath.ClassPathFactory$.newClassPath(ClassPathFactory.scala:91) at scala.tools.nsc.Global.$anonfun$extendCompilerClassPath$1(Global.scala:825) at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238) at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36) at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33) at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38) at scala.collection.TraversableLike.map(TraversableLike.scala:238) at scala.collection.TraversableLike.map$(TraversableLike.scala:231) at scala.collection.AbstractTraversable.map(Traversable.scala:108) at scala.tools.nsc.Global.extendCompilerClassPath(Global.scala:825) at scala.tools.nsc.interpreter.IMain.addUrlsToClassPath(IMain.scala:259) at org.apache.kyuubi.engine.spark.operation.ExecuteScala.$anonfun$runInternal$1(ExecuteScala.scala:69) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at org.apache.kyuubi.engine.spark.operation.SparkOperation.withLocalProperties(SparkOperation.scala:88) at org.apache.kyuubi.engine.spark.operation.ExecuteScala.runInternal(ExecuteScala.scala:60) at org.apache.kyuubi.operation.AbstractOperation.run(AbstractOperation.scala:133) at org.apache.kyuubi.session.AbstractSession.runOperation(AbstractSession.scala:94) at org.apache.kyuubi.engine.spark.session.SparkSessionImpl.runOperation(SparkSessionImpl.scala:66) at org.apache.kyuubi.session.AbstractSession.$anonfun$executeStatement$1(AbstractSession.scala:124) at org.apache.kyuubi.session.AbstractSession.withAcquireRelease(AbstractSession.scala:75) at org.apache.kyuubi.session.AbstractSession.executeStatement(AbstractSession.scala:121) at org.apache.kyuubi.service.AbstractBackendService.executeStatement(AbstractBackendService.scala:66) at org.apache.kyuubi.service.TFrontendService.ExecuteStatement(TFrontendService.scala:232) at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1557) at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1542) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:38) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.kyuubi.service.authentication.TSetIpAddressProcessor.process(TSetIpAddressProcessor.scala:36) at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:310) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
22/06/20 17:14:27 INFO ExecuteScala: Processing 167484's query[f345acf0-4dae-4b5f-a836-abc6a5672c55]: RUNNING_STATE -> ERROR_STATE, statement: import com.datastax.spark.connector.cql.CassandraConnectorConf, time taken: 0.018 seconds 22/06/20 17:14:27 ERROR SparkTBinaryFrontendService: Error executing statement: org.apache.kyuubi.KyuubiSQLException: Error operating EXECUTE_STATEMENT: java.lang.NullPointerException at scala.tools.nsc.classpath.FileUtils$AbstractFileOps$.isJarOrZip$extension(FileUtils.scala:32) at scala.tools.nsc.classpath.ClassPathFactory$.newClassPath(ClassPathFactory.scala:91) at scala.tools.nsc.Global.$anonfun$extendCompilerClassPath$1(Global.scala:825) at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238) at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36) at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33) at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38) at scala.collection.TraversableLike.map(TraversableLike.scala:238) at scala.collection.TraversableLike.map$(TraversableLike.scala:231) at scala.collection.AbstractTraversable.map(Traversable.scala:108) at scala.tools.nsc.Global.extendCompilerClassPath(Global.scala:825) at scala.tools.nsc.interpreter.IMain.addUrlsToClassPath(IMain.scala:259) at org.apache.kyuubi.engine.spark.operation.ExecuteScala.$anonfun$runInternal$1(ExecuteScala.scala:69) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at org.apache.kyuubi.engine.spark.operation.SparkOperation.withLocalProperties(SparkOperation.scala:88) at org.apache.kyuubi.engine.spark.operation.ExecuteScala.runInternal(ExecuteScala.scala:60) at org.apache.kyuubi.operation.AbstractOperation.run(AbstractOperation.scala:133) at org.apache.kyuubi.session.AbstractSession.runOperation(AbstractSession.scala:94) at org.apache.kyuubi.engine.spark.session.SparkSessionImpl.runOperation(SparkSessionImpl.scala:66) at org.apache.kyuubi.session.AbstractSession.$anonfun$executeStatement$1(AbstractSession.scala:124) at org.apache.kyuubi.session.AbstractSession.withAcquireRelease(AbstractSession.scala:75) at org.apache.kyuubi.session.AbstractSession.executeStatement(AbstractSession.scala:121) at org.apache.kyuubi.service.AbstractBackendService.executeStatement(AbstractBackendService.scala:66) at org.apache.kyuubi.service.TFrontendService.ExecuteStatement(TFrontendService.scala:232) at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1557) at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1542) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:38) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.kyuubi.service.authentication.TSetIpAddressProcessor.process(TSetIpAddressProcessor.scala:36) at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:310) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) at org.apache.kyuubi.KyuubiSQLException$.apply(KyuubiSQLException.scala:69) at org.apache.kyuubi.engine.spark.operation.SparkOperation$$anonfun$onError$1.applyOrElse(SparkOperation.scala:113) at org.apache.kyuubi.engine.spark.operation.SparkOperation$$anonfun$onError$1.applyOrElse(SparkOperation.scala:97) at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:38) at org.apache.kyuubi.engine.spark.operation.ExecuteScala.$anonfun$runInternal$1(ExecuteScala.scala:89) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at org.apache.kyuubi.engine.spark.operation.SparkOperation.withLocalProperties(SparkOperation.scala:88) at org.apache.kyuubi.engine.spark.operation.ExecuteScala.runInternal(ExecuteScala.scala:60) at org.apache.kyuubi.operation.AbstractOperation.run(AbstractOperation.scala:133) at org.apache.kyuubi.session.AbstractSession.runOperation(AbstractSession.scala:94) at org.apache.kyuubi.engine.spark.session.SparkSessionImpl.runOperation(SparkSessionImpl.scala:66) at org.apache.kyuubi.session.AbstractSession.$anonfun$executeStatement$1(AbstractSession.scala:124) at org.apache.kyuubi.session.AbstractSession.withAcquireRelease(AbstractSession.scala:75) at org.apache.kyuubi.session.AbstractSession.executeStatement(AbstractSession.scala:121) at org.apache.kyuubi.service.AbstractBackendService.executeStatement(AbstractBackendService.scala:66) at org.apache.kyuubi.service.TFrontendService.ExecuteStatement(TFrontendService.scala:232) at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1557) at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1542) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:38) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.kyuubi.service.authentication.TSetIpAddressProcessor.process(TSetIpAddressProcessor.scala:36) at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:310) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.NullPointerException at scala.tools.nsc.classpath.FileUtils$AbstractFileOps$.isJarOrZip$extension(FileUtils.scala:32) at scala.tools.nsc.classpath.ClassPathFactory$.newClassPath(ClassPathFactory.scala:91) at scala.tools.nsc.Global.$anonfun$extendCompilerClassPath$1(Global.scala:825) at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238) at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36) at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33) at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38) at scala.collection.TraversableLike.map(TraversableLike.scala:238) at scala.collection.TraversableLike.map$(TraversableLike.scala:231) at scala.collection.AbstractTraversable.map(Traversable.scala:108) at scala.tools.nsc.Global.extendCompilerClassPath(Global.scala:825) `
Kyuubi Server Configurations
No response
Kyuubi Engine Configurations
Additional context
No response
Are you willing to submit PR?
- [ ] Yes I am willing to submit a PR!
