spark spark-shell报错

柔光的暖阳◎ 2024-02-18 18:25 30阅读 0赞

相关版本：spark-1.6.0-cdh5.7.0

![70][]

spark-shell --master yarn --jars /home/hadoop/hive/lib/mysql-connector-java-5.1.44.jar 报错

18/03/01 11:27:44 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable  
java.lang.NoClassDefFoundError: com/fasterxml/jackson/databind/Module  
at java.lang.Class.forName0(Native Method)  
at java.lang.Class.forName(Class.java:348)  
at org.apache.spark.util.Utils$.classForName(Utils.scala:175)  
at org.apache.spark.metrics.MetricsSystem$$anonfun$registerSinks$1.apply(MetricsSystem.scala:190)  
at org.apache.spark.metrics.MetricsSystem$$anonfun$registerSinks$1.apply(MetricsSystem.scala:186)  
at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)  
at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)  
at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:226)  
at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:39)  
at scala.collection.mutable.HashMap.foreach(HashMap.scala:98)  
at org.apache.spark.metrics.MetricsSystem.registerSinks(MetricsSystem.scala:186)  
at org.apache.spark.metrics.MetricsSystem.start(MetricsSystem.scala:100)  
at org.apache.spark.SparkContext.<init>(SparkContext.scala:551)  
at org.apache.spark.repl.SparkILoop.createSparkContext(SparkILoop.scala:1022)  
at $iwC$$iwC.<init>(<console>:15)  
at $iwC.<init>(<console>:25)  
at <init>(<console>:27)  
at .<init>(<console>:31)  
at .<clinit>(<console>)  
at .<init>(<console>:7)  
at .<clinit>(<console>)  
at $print(<console>)  
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  
at java.lang.reflect.Method.invoke(Method.java:498)  
at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1045)  
at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1326)  
at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:821)  
at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:852)  
at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:800)  
at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:857)  
at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902)  
at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814)  
at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:125)  
at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:124)  
at org.apache.spark.repl.SparkIMain.beQuietDuring(SparkIMain.scala:305)  
at org.apache.spark.repl.SparkILoopInit$class.initializeSpark(SparkILoopInit.scala:124)  
at org.apache.spark.repl.SparkILoop.initializeSpark(SparkILoop.scala:64)  
at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1$$anonfun$apply$mcZ$sp$5.apply$mcV$sp(SparkILoop.scala:974)  
at org.apache.spark.repl.SparkILoopInit$class.runThunks(SparkILoopInit.scala:160)  
at org.apache.spark.repl.SparkILoop.runThunks(SparkILoop.scala:64)  
at org.apache.spark.repl.SparkILoopInit$class.postInitialization(SparkILoopInit.scala:108)  
at org.apache.spark.repl.SparkILoop.postInitialization(SparkILoop.scala:64)  
at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply$mcZ$sp(SparkILoop.scala:991)  
at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)  
at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)  
at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)  
at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$process(SparkILoop.scala:945)  
at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1064)  
at org.apache.spark.repl.Main$.main(Main.scala:31)  
at org.apache.spark.repl.Main.main(Main.scala)  
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  
at java.lang.reflect.Method.invoke(Method.java:498)  
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:731)  
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)  
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)  
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)  
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)  
Caused by: java.lang.ClassNotFoundException: com.fasterxml.jackson.databind.Module  
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)  
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)  
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)  
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)  
... 61 more  
  
  
java.lang.NullPointerException  
at org.apache.spark.sql.SQLContext$.createListenerAndUI(SQLContext.scala:1375)  
at org.apache.spark.sql.SQLContext.<init>(SQLContext.scala:77)  
at org.apache.spark.repl.SparkILoop.createSQLContext(SparkILoop.scala:1038)  
at $iwC$$iwC.<init>(<console>:15)  
at $iwC.<init>(<console>:24)  
at <init>(<console>:26)  
at .<init>(<console>:30)  
at .<clinit>(<console>)  
at .<init>(<console>:7)  
at .<clinit>(<console>)  
at $print(<console>)  
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  
at java.lang.reflect.Method.invoke(Method.java:498)  
at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1045)  
at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1326)  
at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:821)  
at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:852)  
at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:800)  
at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:857)  
at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902)  
at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814)  
at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:133)  
at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:124)  
at org.apache.spark.repl.SparkIMain.beQuietDuring(SparkIMain.scala:305)  
at org.apache.spark.repl.SparkILoopInit$class.initializeSpark(SparkILoopInit.scala:124)  
at org.apache.spark.repl.SparkILoop.initializeSpark(SparkILoop.scala:64)  
at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1$$anonfun$apply$mcZ$sp$5.apply$mcV$sp(SparkILoop.scala:974)  
at org.apache.spark.repl.SparkILoopInit$class.runThunks(SparkILoopInit.scala:160)  
at org.apache.spark.repl.SparkILoop.runThunks(SparkILoop.scala:64)  
at org.apache.spark.repl.SparkILoopInit$class.postInitialization(SparkILoopInit.scala:108)  
at org.apache.spark.repl.SparkILoop.postInitialization(SparkILoop.scala:64)  
at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply$mcZ$sp(SparkILoop.scala:991)  
at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)  
at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)  
at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)  
at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$process(SparkILoop.scala:945)  
at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1064)  
at org.apache.spark.repl.Main$.main(Main.scala:31)  
at org.apache.spark.repl.Main.main(Main.scala)  
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  
at java.lang.reflect.Method.invoke(Method.java:498)  
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:731)  
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)  
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)  
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)  
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)  
  
  
<console>:16: error: not found: value sqlContext  
import sqlContext.implicits.\_  
^  
<console>:16: error: not found: value sqlContext

import sqlContext.sql

解决 加classpath 相关jackson jar包

改spark conf 目录 下 spark-defaults.conf

spark.driver.extraClassPath/home/hadoop/spark/lib/jackson-annotations-2.2.3.jar:/home/hadoop/spark/lib/jackson-core-2.2.3.jar:/home/hadoop/spark/lib/jackson-databind-2.2.3.jar:/home/hadoop/spark/lib/jackson-jaxrs-1.9.2.jar:/home/hadoop/spark/lib/jackson-xc-1.9.2.jar:/home/hadoop/spark/lib/parquet-hadoop-1.4.3.jar

[70]: https://image.dandelioncloud.cn/pgy_files/images/2024/01/29/1ed347d60b7949e9a5de5921bc56a331.png