【Spark笔记】Windows10 本地搭建单机版Spark开发环境

 1 <?xml version="1.0" encoding="UTF-8"?>
 2 <project xmlns="http://maven.apache.org/POM/4.0.0"
 3          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 4          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 5     <modelVersion>4.0.0</modelVersion>
 6 
 7     <groupId>com.hackhan.demo</groupId>
 8     <artifactId>jacklee</artifactId>
 9     <version>1.0-SNAPSHOT</version>
10 
11 
12     <properties>
13         <spark.version>2.3.0</spark.version>
14         <scala.version>2.11</scala.version>
15     </properties>
16 
17     <dependencies>
18         <dependency>
19             <groupId>org.apache.spark</groupId>
20             <artifactId>spark-core_${scala.version}</artifactId>
21             <version>${spark.version}</version>
22         </dependency>
23         <dependency>
24             <groupId>org.apache.spark</groupId>
25             <artifactId>spark-streaming_${scala.version}</artifactId>
26             <version>${spark.version}</version>
27         </dependency>
28 
29         <dependency>
30             <groupId>org.apache.spark</groupId>
31             <artifactId>spark-sql_${scala.version}</artifactId>
32             <version>${spark.version}</version>
33         </dependency>
34         <dependency>
35             <groupId>org.apache.spark</groupId>
36             <artifactId>spark-hive_${scala.version}</artifactId>
37             <version>${spark.version}</version>
38         </dependency>
39         <dependency>
40             <groupId>org.apache.spark</groupId>
41             <artifactId>spark-mllib_${scala.version}</artifactId>
42             <version>${spark.version}</version>
43         </dependency>
44 
45     </dependencies>
46 
47     <build>
48         <plugins>
49 
50             <plugin>
51                 <groupId>org.scala-tools</groupId>
52                 <artifactId>maven-scala-plugin</artifactId>
53                 <version>2.15.2</version>
54                 <executions>
55                     <execution>
56                         <goals>
57                             <goal>compile</goal>
58                             <goal>testCompile</goal>
59                         </goals>
60                     </execution>
61                 </executions>
62             </plugin>
63 
64             <plugin>
65                 <groupId>org.apache.maven.plugins</groupId>
66                 <artifactId>maven-compiler-plugin</artifactId>
67                 <version>3.6.0</version>
68                 <configuration>
69                     <source>1.8</source>
70                     <target>1.8</target>
71                 </configuration>
72             </plugin>
73 
74             <plugin>
75                 <groupId>org.apache.maven.plugins</groupId>
76                 <artifactId>maven-surefire-plugin</artifactId>
77                 <version>2.19</version>
78                 <configuration>
79                     <skip>true</skip>
80                 </configuration>
81             </plugin>
82 
83         </plugins>
84     </build>
85 
86 </project>

 1 package com.hackhan.demo
 2 
 3 import org.apache.spark.{SparkConf, SparkContext}
 4 
 5 
 6 /**
 7   *
 8   * @author migu-orrin on 2018/5/3.
 9   */
10 object WordCount {
11   def main(args: Array[String]) {
12 
13     /**
14       * SparkContext 的初始化需要一个SparkConf对象
15       * SparkConf包含了Spark集群的配置的各种参数
16       */
17     val conf=new SparkConf()
18       .setMaster("local")//启动本地化计算
19       .setAppName("WordCount")//设置本程序名称
20 
21     //Spark程序的编写都是从SparkContext开始的
22     val sc=new SparkContext(conf)
23     //以上的语句等价与val sc=new SparkContext("local","testRdd")
24     val data=sc.textFile("C:/SelfFiles/Spark/test/wordcount.txt")//读取本地文件
25     var result = data.flatMap(_.split(" "))//下划线是占位符，flatMap是对行操作的方法，对读入的数据进行分割
26       .map((_,1))//将每一项转换为key-value，数据是key，value是1
27       .reduceByKey(_+_)//将具有相同key的项相加合并成一个
28 
29     result.collect()//将分布式的RDD返回一个单机的scala array，在这个数组上运用scala的函数操作，并返回结果到驱动程序
30       .foreach(println)//循环打印
31 
32     Thread.sleep(10000)
33     result.saveAsTextFile("C:/SelfFiles/Spark/test/wordcountres")
34     println("OK,over!")
35   }
36 }

this is my first test.