#!/bin/bash
NUM_PARTITIONS=100
if [ $# == 1 ]; then
  PROCESS_DATE=$1
elif [ $# == 2 ]; then
  PROCESS_DATE=$1
  NUM_PARTITIONS=$2
else
  PROCESS_DATE=`date -d "1 day ago" "+%Y%m%d"`

# 检测 redis 中数据信息
  KEY='nrmv:hdfs:'$PROCESS_DATE

  echo 'get '$KEY

  REDIS_RESULT=`redis-cli -h redis-01 -p 63791 -n 1 get $KEY | awk '{print $1}'`

  echo 'REDIS_RESULT: '${REDIS_RESULT:='(Nil)'}

  if [ ${REDIS_RESULT:='(Nil)'} = "0" ]; then
    echo 'active project'
  else
    exit 0
  fi
fi

echo "processDate is "${PROCESS_DATE}

#检测文件目录是否存在

year=${PROCESS_DATE:0:4}

month=${PROCESS_DATE:4:2}

day=${PROCESS_DATE:6:2}

history=/spark/nrmv_history/

dir=${history}${year}"/"${month}"/"${day}

EXECUTOR_CORES=1

NUM_EXECUTORS=5

EXECUTOR_MEMORY=8G

DRIVER_MEMORY=3G

JAR_PATH=../lib/uml-azkaban-history-nd-1.0.0.jar

MAIN_CLASS=com.uml.tech.SparkApp

for jar in ../lib/*.jar
do
  if [ ${jar} != ${JAR_PATH} ] ; then
    LIBJARS=${jar},${LIBJARS}
  fi
done

APP_NAME="azkaban-history-nd."${PROCESS_DATE}

echo "app name : "${APP_NAME}

spark-submit \
  --master yarn \
  --deploy-mode cluster \
  --class ${MAIN_CLASS} \
  --jars ${LIBJARS} \
  --driver-memory ${DRIVER_MEMORY} \
  --executor-cores ${EXECUTOR_CORES} \
  --executor-memory ${EXECUTOR_MEMORY} \
  --num-executors ${NUM_EXECUTORS} \
  --conf spark.app.name=${APP_NAME} \
  --driver-java-options "-XX:+UseG1GC" \
  --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
  --conf spark.streaming.stopGracefullyOnShutdown=true \
  ${JAR_PATH} \
  ${PROCESS_DATE} ${NUM_PARTITIONS}
