wordcount实现的三种方式
第一种方式：
val df1 = spark.sql("select word,count(1) as word_cnt from (select explode(split(sentence, ' ')) as word from badou.wordcount) t group by word order by word_cnt desc")
第二种方式：
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions._
val spark = SparkSession.builder().enableHiveSupport().appName("Word count df").getOrCreate()
val df = spark.sql("select sentence from badou.wordcount")
val split_udf = udf{ sentence:String =>
sentence.split(" ")
}
val wordcount = df.withColumn("split_words",split_udf(col("sentence"))).withColumn("word_explode",explode(col("split_words")))
.groupBy(col("word_explode")).count().withColumnRenamed("count","word_count").orderBy(desc("word_count"))
.select("word_explode","word_count")
wordcount.show()
第三种方式：
import org.apache.spark.sql.{DataFrame, SparkSession}
val spark = SparkSession.builder().appName("Word Count")
.enableHiveSupport()
.getOrCreate()
val df = spark.sql("select sentence from badou.wordcount")
val wordcount_sort = df.select("sentence").rdd.map(x=>x.toString)
.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_)
.sortBy(_._2,ascending = false).foreach(println)
