#!/bin/bash

workFixOutputDir=/spark/farm/fix/work2/

durations=3000

APP_NAME=stream.auto.split.

APP_VERSION=2021.04.27

JAR_PATH=../lib/uml-stream-auto-split-1.0.0.jar

MAIN_CLASS=com.uml.tech.spark.App

executor_cores=1

num_executors=1

executor_memory=2G

for jar in ../lib/*.jar
do
  if [ ${jar} != ${JAR_PATH} ] ; then
    LIBJARS=${jar},${LIBJARS}
  fi
done

config=../resources/application.properties

appId=`yarn application -list | grep ${APP_NAME} | awk '{print $1}'`

if [ "${appId}" != "" ] ; then
  echo `yarn application -kill ${appId}`
fi

echo "start submit spark application ......"

spark-submit \
  --master yarn \
  --deploy-mode cluster \
  --class ${MAIN_CLASS} \
  --conf spark.app.name=${APP_NAME}${APP_VERSION} \
  --jars ${LIBJARS} \
  --files ${config} \
  --executor-cores ${executor_cores} \
  --num-executors ${num_executors} \
  --executor-memory ${executor_memory} \
  --conf spark.driver.memory=1G \
  --conf spark.driver.cores=1 \
  --conf spark.yarn.maxAppAttempts=5 \
  --conf spark.streaming.stopGracefullyOnShutdown=true \
  --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
  --conf "spark.executor.extraJavaOptions= -XX:+UseG1GC"\
  --conf "spark.driver.extraJavaOptions=-XX:+UseG1GC" \
  ${JAR_PATH} \
  ${durations} ${workFixOutputDir}


##--conf spark.executor.extraJavaOptions=" -XX:+UseG1GC" \
##--conf spark.ui.retainedJobs=50 \
##--conf spark.ui.retailedStages=50 \
##--conf spark.worker.ui.retainedExecutors=5 \
## --conf spark.worker.ui.retainedDrivers=5 \
## --conf spark.ui.timeline.tasks.maximum=50 \
## --conf spark.sql.ui.retainedExecutions=50 \
## --conf spark.streaming.ui.retainedBatches=50 \
## --conf spark.streaming.concurrentJobs=4 \