创建Spark Streaming Maven项目

通过IDEA创建一个Maven项目, spark-streaming-demo

编辑pom.xml文件

因为我们使用了CDH 5组件，所以需要引入CDH 5的Maven仓库。

在project标签下面加入

<repositories>
    <repository>
        <id>cloudera</id>
        <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
    </repository>
</repositories>

在properties标签下面添加组件版本信息

<properties>
    <scala.compat.version>2.11</scala.compat.version>
    <spark.version>2.2.3</spark.version>
    <hadoop.version>2.6.0-cdh5.7.0</hadoop.version>
    <hbase.version>1.2.0-cdh5.7.0</hbase.version>
</properties>

在dependencies标签里面添加如下依赖

<!-- Hadoop 依赖-->
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>
    <version>${hadoop.version}</version>
</dependency>

<!-- HBase 依赖 -->
<dependency>
    <groupId>org.apache.hbase</groupId>
    <artifactId>hbase-client</artifactId>
    <version>${hbase.version}</version>
</dependency>

<dependency>
    <groupId>org.apache.hbase</groupId>
    <artifactId>hbase-server</artifactId>
    <version>${hbase.version}</version>
</dependency>

<!-- Spark streaming 依赖 -->

<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-streaming_${scala.compat.version}</artifactId>
    <version>${spark.version}</version>
</dependency>