30 lines
1.2 KiB
Bash
30 lines
1.2 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
pattern='-Dapp.tables.filter=db.table'
|
|
tasks='-Dapp.parallel.tasks=15'
|
|
hdfs='-Dapp.services.hdfs=hdfs://nameservice1:8020'
|
|
tables_owner='-Dapp.tables.owner=owner'
|
|
hive_extra_params='-Dapp.hiveopts.hive.output.file.extension=.snappy.parquet'
|
|
|
|
nohup spark-submit --class ru.sa2.hive.concatenator.Main \
|
|
--conf spark.driver.cores=1 --conf spark.driver.memory=16G \
|
|
--conf spark.yarn.maxAppAttempts=1 \
|
|
--conf spark.driver.extraJavaOptions="${pattern} ${tasks} ${hdfs} ${tables_owner} ${hive_extra_params}" \
|
|
concatenator-assembly-1.0.jar > concatenator.logs
|
|
|
|
|
|
//////////////////////////////////V2
|
|
#!/usr/bin/env bash
|
|
|
|
pattern='-Dapp.tables.filter=^db.table'
|
|
tasks='-Dapp.parallel.tasks=2'
|
|
hdfs='-Dapp.services.hdfs=hdfs://nameservice1:8020'
|
|
hive_extra_params='-Dapp.hiveopts.0="hive.output.file.extension=.gz.parquet" -Dapp.hiveopts.1="mapred.job.queue.name=queue"'
|
|
tasks_filter_strategy='-Dapp.task.filter.strategy=default'
|
|
|
|
nohup spark-submit --class ru.sa2.hive.concatenator.Main \
|
|
--conf spark.driver.cores=1 --conf spark.driver.memory=16G \
|
|
--conf spark.yarn.maxAppAttempts=1 \
|
|
--conf spark.driver.extraJavaOptions="${pattern} ${tasks} ${tasks_filter_strategy} ${hdfs} ${hive_extra_params}" \
|
|
concatenator-assembly-1.0.jar > concatenator_jbt.logs
|