#!/bin/bash

conf=application.properties

canOutputDir=/spark/farm_can

workOutputDir=/spark/farm_work
# 30 min
durations=1800000

APP_NAME=stream.hdfs.farm_

APP_VERSION=2020.01.17

executor_cores=1

num_executors=8

executor_memory=3G

JAR_PATH=../lib/uml-stream-hdfs-farm-3.2.0.jar

MAIN_CLASS=com.uml.tech.stream.StreamComputing

for jar in ../lib/*.jar; do
  if [ ${jar} != ${JAR_PATH} ]; then
    LIBJARS=${jar},${LIBJARS}
  fi
done

config=../resources/${conf}

appId=$(yarn application -list | grep ${APP_NAME} | awk '{print $1}')

if [ "${appId}" != "" ]; then
  echo $(yarn application -kill ${appId})
fi

echo "start submit spark application ......"

spark-submit \
  --master yarn \
  --deploy-mode cluster \
  --class ${MAIN_CLASS} \
  --conf spark.app.name=${APP_NAME}${APP_VERSION} \
  --jars ${LIBJARS} \
  --files ${config} \
  --executor-cores ${executor_cores} \
  --num-executors ${num_executors} \
  --executor-memory ${executor_memory} \
  --conf spark.yarn.maxAppAttempts=5 \
  --conf spark.dynamicAllocation.enabled=false \
  --conf spark.streaming.stopGracefullyOnShutdown=true \
  --conf spark.streaming.kafka.maxRatePerPartition=10000 \
  --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
  ${JAR_PATH} \
  ${durations} ${canOutputDir} ${workOutputDir} first
