#!/bin/bash
daytime=`date -d "1 day ago" "+%Y%m%d"`
if [ $# == 1 ] ; then
    daytime=$1
elif [ $# == 2 ] ; then
  daytime=$1
  blackList=$2
else
  echo "输入参数错误 ！"
  exit
fi
echo "dayTime is "${daytime}

# 写Tdengine一批次的条数
batch=50

# 时间差的阈值，单位s
time_threshold=300

JAR_PATH=../lib/azkaban-statistics-farm-tdengine-1.0.0.jar

MAIN_CLASS=uml.tech.bigdata.spark.SparkApp

executor_cores=1

num_executors=4

executor_memory=2G
FILE=../conf/configV2.json
for jar in ../lib/*.jar
do
  if [ ${jar} != ${JAR_PATH} ] ; then
    LIBJARS=${jar},${LIBJARS}
  fi
done
GUAVA_JAR_PATH=guava-30.1.1-jre.jar:httpclient-4.5.13.jar:httpcore-4.4.14.jar
appName="azkaban-statistics-farm-tdengine-"${daytime}

echo "app name : "${appName}

spark-submit \
  --master yarn \
  --deploy-mode cluster \
  --class ${MAIN_CLASS} \
  --conf spark.app.name=${appName} \
  --jars ${LIBJARS} \
  --files ${FILE} \
  --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
  --conf spark.driver.memory=2G \
  --conf spark.yarn.executor.memoryOverhead=1024 \
  --conf spark.speculation=true \
  --executor-cores ${executor_cores} \
  --num-executors ${num_executors} \
  --executor-memory ${executor_memory} \
  --conf spark.driver.extraClassPath=${GUAVA_JAR_PATH} \
  --conf spark.executor.extraClassPath=${GUAVA_JAR_PATH} \
  ${JAR_PATH} \
  ${daytime} ${batch} ${time_threshold} ${blackList}
