Read file lines and pass line values as arguments.

Shell Programming and Scripting
1

Dears,

Need help to implement below requirement

A file (detail.txt)contain :

1st column: Stream					
2nd column: PathAddress			    
3rd column: Counterlimit
4th column: TransactionDateColumn	
5th column: DateType				
6th column: SleepValue				
7th column: Status

Need to write a script where it read each line of the file.
Check the status column (0 Not active, 1 active)
then pass as following column values as an argument.
stream
pathaddress
counterlimit on below script.

#!/bin/sh
DATE=`date +"%Y-%m-%d %H:%M:%S"`
LATENCY ()
{
stream=
path=
counter=0
while [ $counter -le $counterlimit ]
do
	for  files in `ls $path | grep -v .temp | head -3`
	do
		if [[ -f $files ]] && [[ -s $files ]];
		then 
			if ## 4th column of the detail.txt having value 1
			then 
				TT_FIRST=`awk -F ',' 'NR==1{print $1}' $files`
				TT_LAST=`awk -F ',' 'END{print $1}' $files`
				FILENAME=`ls $files | cut -d '/' -f6`
				TIMESTAMP=$( date -r $files +'%s')
				FILE_CR_TIME=$( date +'%Y-%m-%d %H:%M:%S' -r $files)
				TRANS_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +'%s')
				TRANS_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +'%s')
				RECORD_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +"%Y-%m-%d %H:%M:%S")
				RECORD_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +"%Y-%m-%d %H:%M:%S")
				TIME_LAG_FIRST=$(expr $TIMESTAMP - $TRANS_TIME_FIRST)
				TIME_LAG_LAST=$(expr $TIMESTAMP - $TRANS_TIME_LAST)
				DELAY_PROCESS=$((`expr $(date -u +"%s") - $TIMESTAMP`))
				TRANS_DIFFTIME=$(expr $TRANS_TIME_LAST - $TRANS_TIME_FIRST)
				counter=$counterlimit
			else
			if   ## 4th column of the detail.txt having value 4
			then
				TTHEX_FIRST=`awk -F ',' 'NR==1{print $4}' $files`
				TTHEX_LAST=`awk -F ',' 'END{print $4}' $files`
				TIMESTAMP=$( date -r $files +'%s')
				FILENAME=`ls $files | cut -d '/' -f5`
				FILE_CR_TIME=$( date +'%Y-%m-%d %H:%M:%S' -r $files)
				TRANS_TIME_FIRST=$(date -d @$(expr `printf "%d" 0x$TTHEX_FIRST` / 1000) +'%s')
				TRANS_TIME_LAST=$(date -d @$(expr `printf "%d" 0x$TTHEX_LAST` / 1000) +'%s')
				RECORD_TIME_FIRST=$(date -d  @$(expr `printf "%d" 0x$TTHEX_FIRST` / 1000) +"%Y-%m-%d %H:%M:%S")
				RECORD_TIME_LAST=$(date -d  @$(expr `printf "%d" 0x$TTHEX_FIRST` / 1000) +"%Y-%m-%d %H:%M:%S")
				TIME_LAG_FIRST=$(expr $TIMESTAMP - $TRANS_TIME_FIRST)
				TIME_LAG_LAST=$(expr $TIMESTAMP - $TRANS_TIME_LAST)
				DELAY_PROCESS=$((`expr $(date -u +"%s") - $TIMESTAMP`))
				TRANS_DIFFTIME=$(expr $TRANS_TIME_LAST - $TRANS_TIME_FIRST)
				counter=100
			else
				echo "file doesn't exist" >/dev/null
			fi
			fi
		if [ "${FILENAME}" != "" ];
		then
			echo "${DATE} ${stream} ${FILENAME} ${FILE_CR_TIME} ${RECORD_TIME_FIRST} ${TIME_LAG_FIRST} ${RECORD_TIME_LAST} ${TIME_LAG_LAST} ${DELAY_PROCESS} ${TRANS_DIFFTIME}" | hdfs dfs -appendToFile - /AUDIT/FINAL.csv
		else
			echo "Filename is empty" >/dev/null 
		fi 
	done
((counter++))
sleep $sleepValue
done
}

detail.txt contains:

S1	/streamslz/LZ/S1	100	1		1	10	1
IUCS	/streamslz/LZ/AnritsuIUCS	100	4	2	5 1
IUPS	/streamslz/LZ/IUPS		100	1	1	5	1

Regards,
sadique

2

I'm not sure what all you need, but hopefully this examples give some ideas for your needs.

1st I made some example file. I hope that if you use some input files, you give those also. Much faster to understand what you are looking.

My script include /bin/somesh.
Somesh = bash or ksh. [ksh source](GitHub - att/ast at beta ksh source)

#!/bin/somesh
###########################################################
# make some example files
# example file
cat <<EOF > detail.txt
S1      /streamslz/LZ/S1        100     1               1       10      0
IUCS    /streamslz/LZ/AnritsuIUCS       100     4       2       5 0
IUPS    /streamslz/LZ/IUPS              100     1       1       5       0
IUPS    /tmp/example            2       1       1       5       1
EOF

mkdir -p /tmp/example 2>/dev/null
cat <<EOF > /tmp/example/1.txt
2017-11-20 12:01:00,2,3,4,5,6,7,8,
2017-11-20 13:01:00,3,3,3,3,3,33,
2017-11-20 14:01:00,98,97,96,95,94
EOF
cat <<EOF > /tmp/example/2.txt
2017-11-21 10:01:00,22,3,4,5,6,7,8,
2017-11-21 11:02:00,3,3,3,3,3,33,
2017-11-21 12:03:00,298,297,296,295,294
EOF

In this example I have used main to handle detail.txt and then call some function after testing status.
I use shell builtin properties to use time and use find to print filelist with timestamp.

#!/bin/somesh
###########################################################
# timestamp
DATE=$( printf "%(%Y-%m-%d %H:%M:%S)T" now )

###########################################################
dosome1()
{
        :
}

###########################################################
dosome2()
{
        # copy args to the variables
        stream="$1"
        path="$2"
        counterlimit="$3"
        datecol="$4"
        sleepinterval="$5"

        counter=0

        CDIR=$PWD
        while (( counter <= counterlimit ))
        do
                cnt=0

               [ ! -d "$path" ] && echo "no $path " >&2 && continue
                cd "$path" || continue # some problem to change dir

                find . -maxdepth 1 -name "*" -type f -printf "%p %TY-%Tm-%Td %TH:%TM:%TS\n" | sort | \
                while read files FILE_CR_TIME
                do
                        [ ! -r "$files" ] && echo "no read access $files" >&2 && continue  # no read access
                        ((cnt+=1))

                        ((cnt>3)) && break # only 3 files / dir

                        FILENAME="${files##*/}"
                        TIMESTAMP=$(printf "%(%#)T" "$FILE_CR_TIME")
                        #echo "FN:$FILENAME epoc:$TIMESTAMP modtime:$FILE_CR_TIME"


                        IFS="," read F1_FLD1 F1_FLD2 F1_FLD3 F1_FLD4 F1_ENDFLDS < $files
                        IFS="," read FL_FLD1 FL_FLD2 FL_FLD3 FL_FLD4 FL_ENDFLDS <<-EOF
                                 $(tail -1 $files)
                                EOF

                        #echo "$FILENAME:$FILENAME TT_FIRST:$F1_FLD1 TT_LAST:$FL_FLD1"
                        case "$datecol" in
                                1)
                                        TT_FIRST=$F1_FLD1
                                        TT_LAST=$FL_FLD1
                                        TRANS_TIME_FIRST=$(printf "%(%#)T" "$TT_FIRST")
                                        TRANS_TIME_LAST=$(printf "%(%#)T" "$TT_LAST")
                                        RECORD_TIME_FIRST=$(printf "%(%Y-%m-%d %H:%M:%S)T" "$TT_FIRST")
                                        RECORD_TIME_LAST=$(printf "%(%Y-%m-%d %H:%M:%S)T" "$TT_LAST")
                                        ((TIME_LAG_FIRST= TIMESTAMP - TRANS_TIME_FIRST))
                                        ((TIME_LAG_LAST= TIMESTAMP - TRANS_TIME_LAST))
                                        #d=$( printf "%(%#)T" now )
                                        ((DELAY_PROCESS = $( printf "%(%#)T" now )  - TIMESTAMP))
                                        ((TRANS_DIFFTIME=TRANS_TIME_LAST - TRANS_TIME_FIRST))
                                        counter=$counterlimit
                                        ;;
                                4)
                                        TTHEX_FIRST=$F1_FLD4
                                        TTHEX_LAST=$FL_FLD4
                                        # ....
                                        counter=100
                                        ;;
                        esac
                        echo "$cnt $DATE $stream $FILENAME $FILE_CR_TIME $RECORD_TIME_FIRST $TIME_LAG_FIRST $RECORD_TIME_LAST $TIME_LAG_LAST $DELAY_PROCESS $TRANS_DIFFTIME"
                done
                ((counter+=1))
                cd $CDIR
                sleep $sleepinterval
        done
}



###########################################################
# MAIN
###########################################################

while read stream pathaddress counterlimit transactiondatecolumn datetype sleepvalue status extraflds
do
        case "$status" in
                0) # not active
                        #echo dosome1 "$stream" "$pathaddress" "$counterlimit" "$transactiondatecolumn" "$datetype"
                        dosome1 "$stream" "$pathaddress" "$counterlimit"  "$transactiondatecolumn" "$datetype" 10
                        ;;
                1) # active
                        #echo dosome2 "$stream" "$pathaddress" "$counterlimit" "$transactiondatecolumn" "$datetype"
                        dosome2 "$stream" "$pathaddress" "$counterlimit"  "$transactiondatecolumn" "$datetype" 10
                        ;;
        esac
done < detail.txt