parquet文件是一个压缩格式

package spark;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.SaveMode;

public class createparquet {
public static void main(String[] args) {
 SparkConf conf = new SparkConf();
 conf.setMaster("local").setAppName("parquet");
 JavaSparkContext sc = new JavaSparkContext(conf);
 SQLContext sqlcontext = new SQLContext(sc);
 JavaRDD<String>jsonrdd = sc.textFile("json");
 DataFrame df=sqlcontext.read().json(jsonrdd);
//	DataFrame dfsqlcontext.read().format("json").load("json");
 //保存成parquet文件
 df.write().mode(SaveMode.Overwrite).format("parquet").save("parquet");
//	df.write().mode(SaveMode.Overwrite).parquet("parquet");
 
 //读取parquet文件
 DataFrame load = sqlcontext.read().format("parquet").load("parquet");
//	load=sqlcontext.read().parquet("parquet");
 load.show();
 
 sc.stop();
}
}

分享到：

赞(0) 打赏