[powershell] load_process_layer_pim_impala.sh
Viewer
*** This page was generated with the meta tag "noindex, nofollow". This happened because you selected this option before saving or the system detected it as spam. This means that this page will never get into the search engines and the search bot will not crawl it. There is nothing to worry about, you can still share it with anyone.
- #!/bin/bash
- #set -x
- #whoami
- ##########################################################################################################
- # Name: load_inges_tables.sh
- # Desc:
- # Sqoop utility script to import column details of RDBMS into DB2 Metadata tables
- # Example : Command line > sh load_process_layer_hive.sh hive_script_name hive_db src_table_name src_table_name appl_cd APPL_CMP_NM Batch_type_cd
- # Example : Command line > sh load_process_layer_hive.sh hive_load_rdx_ing.hql ctmctid01_itsecurityp hdp_v_pru_rdx_ing SECREPRT RDX DLY
- # !! HISTORY !!
- # DATE NAME Description of changes
- # 02/21/17 Tamilselvan Created as part of IT expense dashboard project
- ##########################################################################################################
- function print_log
- {
- echo -e `date +"%y/%m/%d %H:%M:%S"`'\t'"${1}"'\t'"${2}"
- }
- function impala_invalidate
- {
- hive_tbl_name=$1
- #Making the tgt_table_name available in imapla
- $IMPALA_CMD$IMPALA_SERVER$RUN_IMP_QUERY "$IMPALA_INVLDT$hive_db.$hive_tbl_name" >> $job_log_file 2>&1
- if [ $? = 0 ]
- then
- {
- print_log "INFO" "Table $hive_tbl_name is available in impala"
- }
- else
- {
- print_log "ERROR" "Table $hive_tbl_name load will not be available in impala"
- }
- fi
- }
- local_pid=$$
- hive_script_name=$1
- hive_db=$2
- src_table_name=$3
- tgt_table_name=$4
- appl_cd=$5
- appl_cmp_nm=$6
- batch_type_cd=$7
- USER=pimdlkfap01
- #tgt_table_name=hdp_$table_name
- APP_HOME=/opt/pimdlkfap01/gbts/pgim
- script_path=$APP_HOME/scripts
- java_path=$APP_HOME/jars
- HQL_PATH=$APP_HOME/hql
- config_path=$APP_HOME/config
- WORK_PATH=$APP_HOME/work
- log_path=$APP_HOME/logs
- metadata_update_jar=$java_path/MetadataUpdate.jar
- job_log_file=$log_path/job_log_${local_pid}.log
- edl_config_file=$config_path/edl_config.cfg
- appl_config_file=$config_path/application_config.cfg
- CLOUDERA_CONFIG_FILE=$config_path/hive_impala_config.cfg
- #Loading Configuration Files
- . ${edl_config_file}
- . ${CLOUDERA_CONFIG_FILE}
- . ${appl_config_file}
- #Creating the batch ID
- curr_batch_id=`/usr/java/latest/bin/java -Xms20m -Xms10000m -jar $metadata_update_jar $metadata_conn_java_prop $metadata_schema batch start $appl_cd $batch_type_cd INPR`
- if [ "${curr_batch_id}" == "Last Batch Still Running" ]
- then
- curr_batch_id=`/usr/java/latest/bin/java -Xms20m -Xms10000m -jar $metadata_update_jar $metadata_conn_java_prop $metadata_schema batch get $appl_cd $batch_type_cd INPR`
- fi
- print_log "INFO" "Current Batch ID:$curr_batch_id"
- #Creating the Job ID
- job_nm="Load_PROCESS_Layer/${appl_cd}/${batch_type_cd}/$tgt_table_name"
- Curr_job_id=`/usr/java/latest/bin/java -Xms20m -Xms10000m -jar $metadata_update_jar $metadata_conn_java_prop $metadata_schema job start $curr_batch_id $job_nm PROCESS INPR` > $job_log_file
- print_log "INFO" "Job ID is successfully created for the Current Job: $job_nm and the Job ID: $Curr_job_id"
- /usr/java/latest/bin/java -Xms20m -Xms10000m -jar $metadata_update_jar $metadata_conn_java_prop $metadata_schema batch update $curr_batch_id $job_nm
- print_log "INFO" "Running $hive_script_name in hive to populate the data for $tgt_table_name"
- #Loading the ingestion tables
- $IMPALA_CMD$IMPALA_SERVER$RUN_FILE $HQL_PATH/$hive_script_name --var=hive_db=$hive_db --var=current_usr=$USER --var=batch_id=$curr_batch_id --var=src_table_name=$src_table_name --var=tgt_table_name=$tgt_table_name >> $job_log_file 2>&1
- stat_cd=$?
- if [ $stat_cd -ne 0 ]
- then
- status="FAIL"
- EMAIL_EVENT_TYPE_CD="PROCESS_LAYER_LOAD_ERROR"
- EVENTMSG="Loading Hive Process table "$tgt_table_name" is Failed. Please check the logfile at ${job_log_file}."
- print_log "ERROR" "${EVENTMSG}"
- sh $script_path/edl_email_notifn.sh $appl_cd $batch_type_cd PROCESS_LAYER_LOAD $EMAIL_EVENT_TYPE_CD "${EVENTMSG}^^HQL File: ${hive_script_name}" $curr_batch_id >> $job_log_file 2>&1
- exit_cd=1
- else
- status="SUCC"
- print_log "INFO" "Hive Process table "$tgt_table_name" is successfully loaded."
- impala_invalidate $tgt_table_name
- exit_cd=0
- fi
- /usr/java/latest/bin/java -Xms20m -Xms10000m -jar $metadata_update_jar $metadata_conn_java_prop $metadata_schema job end $curr_batch_id $job_nm PROCESS $status
- exit $exit_cd
Editor
You can edit this paste and save as new: