#!/bin/bash
if [ $# == 1 ]; then
  TAG=$1
  PROCESS_DATE=`date -d "1 day ago" "+%Y%m%d"`

  if [ $TAG = "Day" ]; then
    KEY='nrmv:hdfs:'$PROCESS_DATE

    echo 'get '$KEY

    REDIS_RESULT=`redis-cli -h redis-01 -p 63791 -n 1 get $KEY | awk '{print $1}'`

    echo 'REDIS_RESULT: '${REDIS_RESULT:='(Nil)'}

    if [ ${REDIS_RESULT:='(Nil)'} = "1" ]; then
      echo 'active project'
    else
      exit 0
    fi
  fi
elif [ $# == 2 ]; then
  TAG=$1
  PROCESS_DATE=$2
else
  echo "place input <tag> [<processDate> default yesterday]"
fi

echo "processDate is "${PROCESS_DATE}

echo "TAG is "${TAG}

#sh dir.sh $PROCESS_DATE

# 分区数
NUM_PARTITIONS=200

EXECUTOR_CORES=1

NUM_EXECUTORS=20

EXECUTOR_MEMORY=8G

DRIVER_MEMORY=3G

JAR_PATH=../lib/uml-azkaban-statistics-nd-1.0.0.jar

MAIN_CLASS=com.uml.tech.SparkApp

for jar in ../lib/*.jar
do
  if [ ${jar} != ${JAR_PATH} ] ; then
    LIBJARS=${jar},${LIBJARS}
  fi
done

APP_NAME="azkaban-statistics-nd-"${TAG}"."${PROCESS_DATE}

echo "app name : "${APP_NAME}

spark-submit \
  --master yarn \
  --deploy-mode cluster \
  --class ${MAIN_CLASS} \
  --jars ${LIBJARS} \
  --driver-memory ${DRIVER_MEMORY} \
  --executor-cores ${EXECUTOR_CORES} \
  --executor-memory ${EXECUTOR_MEMORY} \
  --num-executors ${NUM_EXECUTORS} \
  --conf spark.app.name=${APP_NAME} \
  --driver-java-options "-XX:+UseG1GC" \
  --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
  --conf spark.streaming.stopGracefullyOnShutdown=true \
  ${JAR_PATH} \
  ${TAG} ${PROCESS_DATE} ${NUM_PARTITIONS}