web-dev-qa-db-ja.com

Spark 2.3.0データソースが見つかりませんでした：kafka

CSVを使用してKafkaストリームをセットアップして、Sparkにストリーミングできるようにしようとしています。しかし、

Exception in thread "main" Java.lang.ClassNotFoundException: Failed to find data source: kafka. Please find packages at http://spark.Apache.org/third-party-projects.html

私のコードは次のようになります

import org.Apache.spark.sql.SparkSession
import org.Apache.spark.sql.functions._
import org.Apache.log4j.{Level, Logger}
import org.Apache.spark.sql.execution.streaming.FileStreamSource.Timestamp
import org.Apache.spark.sql.types._

object SpeedTester {
  def main(args: Array[String]): Unit = {

  val spark = SparkSession.builder.master("local[4]").appName("SpeedTester").config("spark.driver.memory", "8g").getOrCreate()
  val rootLogger = Logger.getRootLogger()
  rootLogger.setLevel(Level.ERROR)
  import spark.implicits._
  val mySchema = StructType(Array(
    StructField("incident_id", IntegerType),
    StructField("date", StringType),
    StructField("state", StringType),
    StructField("city_or_county", StringType),
    StructField("n_killed", IntegerType),
    StructField("n_injured", IntegerType)
  ))

  val streamingDataFrame = spark.readStream.schema(mySchema).csv("C:/Users/zoldham/IdeaProjects/flinkpoc/Data/test")
  streamingDataFrame.selectExpr("CAST(incident_id AS STRING) AS key",
  "to_json(struct(*)) AS value").writeStream
    .format("kafka")
    .option("topic", "testTopic")
    .option("kafka.bootstrap.servers", "localhost:9092")
    .option("checkpointLocation", "C:/Users/zoldham/IdeaProjects/flinkpoc/Data")
    .start()

  val df = spark.readStream.format("kafka").option("kafka.bootstrap.servers", "localhost:9092")
    .option("subscribe", "testTopic").load()
  val df1 = df.selectExpr("CAST(value AS STRING)", "CAST(timestamp AS TIMESTAMP)").as[(String, Timestamp)]
    .select(from_json(col("value"), mySchema).as("data"), col("timestamp"))
    .select("data.*", "timestamp")
  df1.writeStream
    .format("console")
    .option("truncate","false")
    .start()
    .awaitTermination()
  }
}

と私 build.sbtファイルは次のようになります

name := "Spark POC"

version := "0.1"

scalaVersion := "2.11.12"

libraryDependencies += "org.Apache.spark" %% "spark-sql" % "2.3.0"

libraryDependencies += "com.Microsoft.sqlserver" % "mssql-jdbc" % "6.2.1.jre8"

libraryDependencies += "org.scalafx" %% "scalafx" % "8.0.144-R12"

libraryDependencies += "org.Apache.ignite" % "ignite-core" % "2.5.0"

libraryDependencies += "org.Apache.ignite" % "ignite-spring" % "2.5.0"

libraryDependencies += "org.Apache.ignite" % "ignite-indexing" % "2.5.0"

libraryDependencies += "org.Apache.spark" %% "spark-streaming-kafka-0-10_2.11" % "2.3.0"

libraryDependencies += "org.Apache.kafka" % "kafka-clients" % "0.11.0.1"

そのエラーの原因は何ですか？ご覧のとおり、ライブラリの依存関係にKafkaを含め、 公式ガイド にも従いました。スタックトレースは次のとおりです。

Exception in thread "main" Java.lang.ClassNotFoundException: Failed to find data source: kafka. Please find packages at http://spark.Apache.org/third-party-projects.html
    at org.Apache.spark.sql.execution.datasources.DataSource$.lookupDataSource(DataSource.scala:635)
    at org.Apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:283)
    at SpeedTester$.main(SpeedTester.scala:61)
    at SpeedTester.main(SpeedTester.scala)
Caused by: Java.lang.ClassNotFoundException: kafka.DefaultSource
    at Java.net.URLClassLoader.findClass(URLClassLoader.Java:381)
    at Java.lang.ClassLoader.loadClass(ClassLoader.Java:424)
    at Sun.misc.Launcher$AppClassLoader.loadClass(Launcher.Java:349)
    at Java.lang.ClassLoader.loadClass(ClassLoader.Java:357)
    at org.Apache.spark.sql.execution.datasources.DataSource$$anonfun$23$$anonfun$apply$15.apply(DataSource.scala:618)
    at org.Apache.spark.sql.execution.datasources.DataSource$$anonfun$23$$anonfun$apply$15.apply(DataSource.scala:618)
    at scala.util.Try$.apply(Try.scala:192)
    at org.Apache.spark.sql.execution.datasources.DataSource$$anonfun$23.apply(DataSource.scala:618)
    at org.Apache.spark.sql.execution.datasources.DataSource$$anonfun$23.apply(DataSource.scala:618)
    at scala.util.Try.orElse(Try.scala:84)
    at org.Apache.spark.sql.execution.datasources.DataSource$.lookupDataSource(DataSource.scala:618)
    ... 3 more
scalaapache-sparkapache-kafkaspark-structured-streaming
3
Zachary Oldham

不足している依存関係を追加する必要があります

"org.Apache.spark" %% "spark-sql-kafka-0-10" % "2.3.0"

それが述べたように ドキュメントで または ここで 例えば。

4
gemelen