import scala.Tuple2;import org.apache.spark.api.java.JavaPairRDD;import org.apache.spark.api.java.JavaRDD;import org.apache.spark.api.java.function.FlatMapFunction;import org.apache.spark.api.java.function.Function2;import org.apache.spark.api.java.function.PairFunction;import org.apache.spark.sql.Sparksession;import java.util.Arrays;import java.util.Iterator;import java.util.List;import java.util.regex.Pattern;public final class JavaWordCount { PRivate static final Pattern SPACE = Pattern.compile(" "); public static void main(String[] args) throws Exception { //如果没有输入参数，则提示需要文件 if (args.length < 1) { System.err.println("Usage: JavaWordCount <file>"); System.exit(1); } //builder():Creates a SparkSession.Builder for constructing a SparkSession. //appName():Sets a name for the application, which will be shown in the Spark web UI. If no application name is set, a randomly generated name will be used. //getOrCreate():Gets an existing SparkSession or, if there is no existing one, creates a new one based on the options set in this builder. SparkSession spark = SparkSession .builder() .appName("JavaWordCount") .getOrCreate(); //read():Returns a DataFrameReader that can be used to read non-streaming data in as a DataFrame. //textFile():Loads text files and returns a Dataset of String. //javaRDD():Returns the content of the Dataset as a JavaRDD of Ts. JavaRDD<String> lines = spark.read().textFile(args[0]).javaRDD(); //public static <U> JavaRDD<U> flatMap(FlatMapFunction<T,U> f) JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String, String>() { @Override public Iterator<String> call(String s) { //将String分隔并且变成迭代器 return Arrays.asList(SPACE.split(s)).iterator(); } }); //public static <K2,V2> JavaPairRDD<K2,V2> mapToPair(PairFunction<T,K2,V2> f) JavaPairRDD<String, Integer> ones = words.mapToPair( new PairFunction<String, String, Integer>() { @Override public Tuple2<String, Integer> call(String s) { return new Tuple2<String, Integer>(s, 1); } }); //public JavaPairRDD<K,V> reduceByKey(Function2<V,V,V> func) JavaPairRDD<String, Integer> counts = ones.reduceByKey( new Function2<Integer, Integer, Integer>() { @Override public Integer call(Integer i1, Integer i2) { return i1 + i2; } }); //public static java.util.List<T> collect() List<Tuple2<String, Integer>> output = counts.collect(); for (Tuple2<?,?> tuple : output) { System.out.println(tuple._1() + ": " + tuple._2()); } spark.stop(); }}
这里要添加好依赖关系，否则编译不成功
可以没有./spark-warehouse
file:///usr/local/spark/README.md是执行时输入的文件，这个可以修改，看代码可知
