#!/bin/bash
if [ $# == 1 ] ; then
  tag=$1
  daytime=`date -d "1 day ago" "+%Y%m%d"`
elif [ $# == 2 ] ; then
  tag=$1
  daytime=$2
else
  echo "输入参数错误 ！"
  exit
fi

sh dir.sh ${daytime}

echo "tag is "${tag}

echo "dayTime is "${daytime}

numPartitions=100

total=1

JAR_PATH=../lib/uml-azkaban-statistics-ne-1.0.0.jar

MAIN_CLASS=com.uml.tech.spark.SparkApp

executor_cores=1

num_executors=5

executor_memory=12G

for jar in ../lib/*.jar
do
  if [ ${jar} != ${JAR_PATH} ] ; then
    LIBJARS=${jar},${LIBJARS}
  fi
done

appName=azkaban_statistics_ne_${tag}.${daytime}

spark-submit \
  --master yarn \
  --deploy-mode cluster \
  --class ${MAIN_CLASS} \
  --conf spark.app.name=${appName} \
  --jars ${LIBJARS} \
  --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
  --conf spark.shuffle.memoryFraction=0.4 \
  --conf spark.storage.memoryFraction=0.3 \
  --conf spark.driver.memory=8G \
  --executor-cores ${executor_cores} \
  --num-executors ${num_executors} \
  --executor-memory ${executor_memory} \
  ${JAR_PATH} \
  ${tag} ${daytime} ${numPartitions} ${total}