springboot集成spark并使用spark

springboot集成spark并使用spark

首先添加相关依赖：

xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

4.0.0

org.springframework.boot

spring-boot-starter-parent

1.5.6.RELEASE

com.cord

spark-example

1.0-SNAPSHOT

spark-example

http://example.com

UTF-8

UTF-8

1.8

2.10.3

1.8

1.8

org.springframework.boot

spring-boot-starter

1.5.6.RELEASE

org.springframework.boot

spring-boot-starter-logging

org.apache.spark

spark-core_2.10

1.6.1

provided

org.slf4j

slf4j-log4j12

log4j

log4j

org.apache.spark

spark-sql_2.10

1.6.1

provided

org.apache.spark

spark-hive_2.10

1.6.1

provided

org.scala-lang

scala-library

${scala.version}

provided

mysql

mysql-connector-java

5.1.22

org.apache.maven.plugins

maven-shade-plugin

org.springframework.boot

spring-boot-maven-plugin

1.5.6.RELEASE

false

false

*:*

META-INF/*.SF

META-INF/*.DSA

META-INF/*.RSA

implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">

META-INF/spring.handlers

implementation="org.springframework.boot.maven.PropertiesMergingResourceTransformer">

META-INF/spring.factories

implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">

META-INF/spring.schemas

implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />

implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">

com.cord.StartApplication

package

shade

xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

4.0.0

org.springframework.boot

spring-boot-starter-parent

1.5.6.RELEASE

com.cord

spark-example

1.0-SNAPSHOT

spark-example

http://example.com

UTF-8

UTF-8

1.8

2.10.3

1.8

1.8

org.springframework.boot

spring-boot-starter

1.5.6.RELEASE

org.springframework.boot

spring-boot-starter-logging

org.apache.spark

spark-core_2.10

1.6.1

provided

org.slf4j

slf4j-log4j12

log4j

log4j

org.apache.spark

spark-sql_2.10

1.6.1

provided

org.apache.spark

spark-hive_2.10

1.6.1

provided

org.scala-lang

scala-library

${scala.version}

provided

mysql

mysql-connector-java

5.1.22

org.apache.maven.plugins

maven-shade-plugin

org.springframework.boot

spring-boot-maven-plugin

1.5.6.RELEASE

false

false

*:*

META-INF/*.SF

META-INF/*.DSA

META-INF/*.RSA

implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">

META-INF/spring.handlers

implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">

META-INF/spring.handlers

implementation="org.springframework.boot.maven.PropertiesMergingResourceTransformer">

META-INF/spring.factories

implementation="org.springframework.boot.maven.PropertiesMergingResourceTransformer">

META-INF/spring.factories

implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">

META-INF/spring.schemas

implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">

META-INF/spring.schemas

implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />

implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">

com.cord.StartApplication

package

shade

implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />

implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">

com.cord.StartApplication

implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">

com.cord.StartApplication

package

shade

需要注意的是依赖中排除掉的日志模块，以及特殊的打包方式

定义配置类:

SparkContextBean.class

@Configuration

public class SparkContextBean {

private String appName = "sparkExp";

private String master = "local";

@Bean

@ConditionalOnMissingBean(SparkConf.class)

public SparkConf sparkConf() throws Exception {

SparkConf conf = new SparkConf().setAppName(appName).setMaster(master);

return conf;

}

@Bean

@ConditionalOnMissingBean

public JavaSparkContext javaSparkContext() throws Exception {

return new JavaSparkContext(sparkConf());

}

@Bean

@ConditionalOnMissingBean

publiDdTpJc HiveContext hiveContext() throws Exception {

return new HiveContext(javaSparkContext());

}

......

}

启动类：

StartApplication.class

@SpringBootApplication

public class StartApplication implements CommandLineRunner {

@Autowired

private HiveContext hc;

public static void main(String[] args) {

SpringApplication.run(StartApplication.class, args);

}

@Override

public void run(String... args) throws Exception {

DataFrame df = hc.sql("select count(1) from LCS_DB.STAFF_INFO");

List result = df.javaRDD().map((Function) row -> {

return row.getLong(0);

}).collect();

result.stream().forEach(System.out::println);

}

执行方式：

spark-submit \

--class com.cord.StartApplication \

--executor-memory 4G \

--num-executors 8 \

--master yarn-client \

/data/cord/spark-example-1.0-SNAPSHOT.jar

参考链接：

https://stackoverflow.com/questions/45189701/submitting-spring-boot-application-jar-to-spark-submit

https://my.oschina.net/woter/blog/1843755

版权声明：本文内容由网络用户投稿，版权归原作者所有，本站不拥有其著作权，亦不承担相应法律责任。如果您发现本站中有涉嫌抄袭或描述失实的内容，请联系我们jiasou666@gmail.com 处理，核实后本网站将在24小时内删除侵权内容。