#!/bin/bash
if [ $# == 1 ]; then
  PROCESS_DATE=`date -d "1 day ago" "+%Y%m%d"`
  BATCH=$1
elif [ $# == 2 ]; then
  BATCH=$1
  PROCESS_DATE=$2
else
  echo "place input <tag> [<processDate> default yesterday]"
fi

echo "processDate is "${PROCESS_DATE}

echo "BATCH is "${BATCH}

# 分区数
NUM_PARTITIONS=1000

EXECUTOR_CORES=1

NUM_EXECUTORS=60

EXECUTOR_MEMORY=4G

DRIVER_MEMORY=4G

JAR_PATH=../lib/uml-hdfs-to-tdengine-farm-can-nd-1.0.0.jar

MAIN_CLASS=com.uml.tech.spark.StreamingApp

for jar in ../lib/*.jar
do
  if [ ${jar} != ${JAR_PATH} ] ; then
    LIBJARS=${jar},${LIBJARS}
  fi
done

APP_NAME="uml-hdfs-to-tdengine-farm-can-nd-"${TAG}"."${PROCESS_DATE}

echo "app name : "${APP_NAME}

spark-submit \
  --master yarn \
  --deploy-mode cluster \
  --class ${MAIN_CLASS} \
  --jars ${LIBJARS} \
  --driver-memory ${DRIVER_MEMORY} \
  --executor-cores ${EXECUTOR_CORES} \
  --executor-memory ${EXECUTOR_MEMORY} \
  --num-executors ${NUM_EXECUTORS} \
  --conf spark.app.name=${APP_NAME} \
  --driver-java-options "-XX:+UseG1GC" \
  --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
  --conf spark.streaming.stopGracefullyOnShutdown=true \
  ${JAR_PATH} \
  ${PROCESS_DATE} ${NUM_PARTITIONS} ${BATCH}