Shell script to pass the config file lines as variable on the respective called function on a script

Shell Programming and Scripting
1

I want to make a config file which contain all the paths.
i want to read the config file line by line and pass as an argument on my below function.
Replace all the path with reading config path line by line and pass in respective functions.
how can i achieve that?

Kindly guide.


#!/bin/sh
DATE=`date +"%Y-%m-%d %H:%M:%S"`   
S1()
{
stream=S1
path=/sadique/S1
counter=0
while [ $counter -le 100 ]
do
	for  files in `printf "%s\n" $path/*.RTM | head -3`
	do
		if [[ -f $files ]];
		then
		TT_FIRST=`awk -F ',' 'NR==1{print $1}' $files`
		TT_LAST=`awk -F ',' 'END{print $1}' $files`
		FILENAME=`ls $files | cut -d '/' -f6`
		TIMESTAMP=$( date -r $files +'%s')
		FILE_CR_TIME=$( date +'%Y-%m-%d %H:%M:%S' -r $files)
                TRANS_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +'%s')
		TRANS_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +'%s')
		RECORD_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +"%Y-%m-%d %H:%M:%S")
		RECORD_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +"%Y-%m-%d %H:%M:%S")
		TIME_LAG_FIRST=$(expr $TIMESTAMP - $TRANS_TIME_FIRST)
		TIME_LAG_LAST=$(expr $TIMESTAMP - $TRANS_TIME_LAST)
		DELAY_PROCESS=$((`expr $(date -u +"%s") - $TIMESTAMP`))
		TRANS_DIFFTIME=$(expr $TRANS_TIME_LAST - $TRANS_TIME_FIRST)
		counter=100
		else
			echo "file doesn't exist" >/dev/null
		fi	
		if [ "${FILENAME}" != "" ]; 
		then
			echo "${DATE} ${stream} ${FILENAME} ${FILE_CR_TIME} ${RECORD_TIME_FIRST} ${TIME_LAG_FIRST} ${RECORD_TIME_LAST} ${TIME_LAG_LAST} ${DELAY_PROCESS} ${TRANS_DIFFTIME}" | hadoop dfs -appendToFile - /AUDIT/FINAL.csv
		else
			echo "Filename is empty" >/dev/null 
		fi 
	done
((counter++))
sleep 5s
done
}

IUPS()
{
stream=IUPS
path=/sadique/IUPS
counter=0
while [ $counter -le 100 ]
do
	for  files in `printf "%s\n" $path/*.RTM | head -3`
	do
		if [[ -f $files ]];
		then
		TT_FIRST=`awk -F ',' 'NR==1{print $1}' $files`
		TT_LAST=`awk -F ',' 'END{print $1}' $files`
		FILENAME=`ls $files | cut -d '/' -f6`
		TIMESTAMP=$( date -r $files +'%s')
		FILE_CR_TIME=$( date +'%Y-%m-%d %H:%M:%S' -r $files)
                TRANS_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +'%s')
		TRANS_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +'%s')
		RECORD_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +"%Y-%m-%d %H:%M:%S")
		RECORD_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +"%Y-%m-%d %H:%M:%S")
		TIME_LAG_FIRST=$(expr $TIMESTAMP - $TRANS_TIME_FIRST)
		TIME_LAG_LAST=$(expr $TIMESTAMP - $TRANS_TIME_LAST)
		DELAY_PROCESS=$((`expr $(date -u +"%s") - $TIMESTAMP`))
		TRANS_DIFFTIME=$(expr $TRANS_TIME_LAST - $TRANS_TIME_FIRST)
		counter=100
		else
			echo "file doesn't exist" >/dev/null
		fi	
		if [ "${FILENAME}" != "" ]; 
		then
			echo "${DATE} ${stream} ${FILENAME} ${FILE_CR_TIME} ${RECORD_TIME_FIRST} ${TIME_LAG_FIRST} ${RECORD_TIME_LAST} ${TIME_LAG_LAST} ${DELAY_PROCESS} ${TRANS_DIFFTIME}" | hadoop dfs -appendToFile - /AUDIT/FINAL.csv
		else
			echo "Filename is empty" >/dev/null 
		fi 
	done
((counter++))
sleep 5s
done
}

IUCS()
{
stream=IUCS
path=/sadique/IUCS
counter=0
while [ $counter -le 100 ]
do
	for  files in `printf "%s\n" $path/*| head -3`
	do
		if [[ -f $files ]] && [[ -s $files ]];
		then
		TTHEX_FIRST=`awk -F ',' 'NR==1{print $4}' $files`
		TTHEX_LAST=`awk -F ',' 'END{print $4}' $files`
		TIMESTAMP=$( date -r $files +'%s')
		FILENAME=`ls $files | cut -d '/' -f5`
		FILE_CR_TIME=$( date +'%Y-%m-%d %H:%M:%S' -r $files)
		TRANS_TIME_FIRST=$(date -d @$(expr `printf "%d" 0x$TTHEX_FIRST` / 1000) +'%s')
		TRANS_TIME_LAST=$(date -d @$(expr `printf "%d" 0x$TTHEX_LAST` / 1000) +'%s')
		RECORD_TIME_FIRST=$(date -d  @$(expr `printf "%d" 0x$TTHEX_FIRST` / 1000) +"%Y-%m-%d %H:%M:%S")
		RECORD_TIME_LAST=$(date -d  @$(expr `printf "%d" 0x$TTHEX_FIRST` / 1000) +"%Y-%m-%d %H:%M:%S")
		TIME_LAG_FIRST=$(expr $TIMESTAMP - $TRANS_TIME_FIRST)
		TIME_LAG_LAST=$(expr $TIMESTAMP - $TRANS_TIME_LAST)
		DELAY_PROCESS=$((`expr $(date -u +"%s") - $TIMESTAMP`))
		TRANS_DIFFTIME=$(expr $TRANS_TIME_LAST - $TRANS_TIME_FIRST)
		counter=100
		else
			echo "file doesn't exist" >/dev/null
		fi	
		if [ "${FILENAME}" != "" ] && [ "${FILENAME}" != "*.temp" ];
		then
			echo "${DATE} ${stream} ${FILENAME} ${FILE_CR_TIME} ${RECORD_TIME_FIRST} ${TIME_LAG_FIRST} ${RECORD_TIME_LAST} ${TIME_LAG_LAST} ${DELAY_PROCESS} ${TRANS_DIFFTIME}" | hadoop dfs -appendToFile - /AUDIT/FINAL.csv
		else
			echo "Filename is empty" >/dev/null 
		fi 
	done
((counter++))
sleep 5s
done
}

S1 &
IUPS & 
IUCS &

---------- Post updated at 08:22 AM ---------- Previous update was at 05:45 AM ----------

created a file path and placed below:

export input1=/sadique/S1
export input2=/sadique/IUPS
export input3=/sadique/IUCS

then:

#!/bin/sh
DATE=`date +"%Y-%m-%d %H:%M:%S"`   
source /DPI_IN/path

S1()
{
stream=S1
counter=0
while [ $counter -le 100 ]
do
	for  files in `printf "%s\n" ${input1}/*.RTM | head -3`
	do
		if [[ -f $files ]];
		then
		TT_FIRST=`awk -F ',' 'NR==1{print $1}' $files`
		TT_LAST=`awk -F ',' 'END{print $1}' $files`
		FILENAME=`ls $files | cut -d '/' -f6`
		TIMESTAMP=$( date -r $files +'%s')
		FILE_CR_TIME=$( date +'%Y-%m-%d %H:%M:%S' -r $files)
                TRANS_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +'%s')
		TRANS_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +'%s')
		RECORD_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +"%Y-%m-%d %H:%M:%S")
		RECORD_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +"%Y-%m-%d %H:%M:%S")
		TIME_LAG_FIRST=$(expr $TIMESTAMP - $TRANS_TIME_FIRST)
		TIME_LAG_LAST=$(expr $TIMESTAMP - $TRANS_TIME_LAST)
		DELAY_PROCESS=$((`expr $(date -u +"%s") - $TIMESTAMP`))
		TRANS_DIFFTIME=$(expr $TRANS_TIME_LAST - $TRANS_TIME_FIRST)
		counter=100
		else
			echo "file doesn't exist" >/dev/null
		fi	
		if [ "${FILENAME}" != "" ]; 
		then
			echo "${DATE} ${stream} ${FILENAME} ${FILE_CR_TIME} ${RECORD_TIME_FIRST} ${TIME_LAG_FIRST} ${RECORD_TIME_LAST} ${TIME_LAG_LAST} ${DELAY_PROCESS} ${TRANS_DIFFTIME}" | hadoop dfs -appendToFile - /AUDIT/FINAL.csv
		else
			echo "Filename is empty" >/dev/null 
		fi 
	done
((counter++))
sleep 5s
done
}

IUPS()
{
stream=IUPS
path=/sadique/IUPS
counter=0
while [ $counter -le 100 ]
do
	for  files in `printf "%s\n" ${input2}/*.RTM | head -3`
	do
		if [[ -f $files ]];
		then
		TT_FIRST=`awk -F ',' 'NR==1{print $1}' $files`
		TT_LAST=`awk -F ',' 'END{print $1}' $files`
		FILENAME=`ls $files | cut -d '/' -f6`
		TIMESTAMP=$( date -r $files +'%s')
		FILE_CR_TIME=$( date +'%Y-%m-%d %H:%M:%S' -r $files)
                TRANS_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +'%s')
		TRANS_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +'%s')
		RECORD_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +"%Y-%m-%d %H:%M:%S")
		RECORD_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +"%Y-%m-%d %H:%M:%S")
		TIME_LAG_FIRST=$(expr $TIMESTAMP - $TRANS_TIME_FIRST)
		TIME_LAG_LAST=$(expr $TIMESTAMP - $TRANS_TIME_LAST)
		DELAY_PROCESS=$((`expr $(date -u +"%s") - $TIMESTAMP`))
		TRANS_DIFFTIME=$(expr $TRANS_TIME_LAST - $TRANS_TIME_FIRST)
		counter=100
		else
			echo "file doesn't exist" >/dev/null
		fi	
		if [ "${FILENAME}" != "" ]; 
		then
			echo "${DATE} ${stream} ${FILENAME} ${FILE_CR_TIME} ${RECORD_TIME_FIRST} ${TIME_LAG_FIRST} ${RECORD_TIME_LAST} ${TIME_LAG_LAST} ${DELAY_PROCESS} ${TRANS_DIFFTIME}" | hadoop dfs -appendToFile - /AUDIT/FINAL.csv
		else
			echo "Filename is empty" >/dev/null 
		fi 
	done
((counter++))
sleep 5s
done
}

IUCS()
{
stream=IUCS
path=/sadique/IUCS
counter=0
while [ $counter -le 100 ]
do
	for  files in `printf "%s\n" ${input3}/*| head -3`
	do
		if [[ -f $files ]] && [[ -s $files ]];
		then
		TTHEX_FIRST=`awk -F ',' 'NR==1{print $4}' $files`
		TTHEX_LAST=`awk -F ',' 'END{print $4}' $files`
		TIMESTAMP=$( date -r $files +'%s')
		FILENAME=`ls $files | cut -d '/' -f5`
		FILE_CR_TIME=$( date +'%Y-%m-%d %H:%M:%S' -r $files)
		TRANS_TIME_FIRST=$(date -d @$(expr `printf "%d" 0x$TTHEX_FIRST` / 1000) +'%s')
		TRANS_TIME_LAST=$(date -d @$(expr `printf "%d" 0x$TTHEX_LAST` / 1000) +'%s')
		RECORD_TIME_FIRST=$(date -d  @$(expr `printf "%d" 0x$TTHEX_FIRST` / 1000) +"%Y-%m-%d %H:%M:%S")
		RECORD_TIME_LAST=$(date -d  @$(expr `printf "%d" 0x$TTHEX_FIRST` / 1000) +"%Y-%m-%d %H:%M:%S")
		TIME_LAG_FIRST=$(expr $TIMESTAMP - $TRANS_TIME_FIRST)
		TIME_LAG_LAST=$(expr $TIMESTAMP - $TRANS_TIME_LAST)
		DELAY_PROCESS=$((`expr $(date -u +"%s") - $TIMESTAMP`))
		TRANS_DIFFTIME=$(expr $TRANS_TIME_LAST - $TRANS_TIME_FIRST)
		counter=100
		else
			echo "file doesn't exist" >/dev/null
		fi	
		if [ "${FILENAME}" != "" ] && [ "${FILENAME}" != "*.temp" ];
		then
			echo "${DATE} ${stream} ${FILENAME} ${FILE_CR_TIME} ${RECORD_TIME_FIRST} ${TIME_LAG_FIRST} ${RECORD_TIME_LAST} ${TIME_LAG_LAST} ${DELAY_PROCESS} ${TRANS_DIFFTIME}" | hadoop dfs -appendToFile - /AUDIT/FINAL.csv
		else
			echo "Filename is empty" >/dev/null 
		fi 
	done
((counter++))
sleep 5s
done
}

S1 &
IUPS & 
IUCS &
2

Can you condense this to a suitable working example to simply illustrate your question? There is too much to consider here in one go.

Thanks,
Robin

1 Like
3 
S1()
{
stream=S1
path=/sadique/S1
counter=0
while [ $counter -le 100 ]
do
	for  files in `printf "%s\n" $path/*.RTM | head -3`
	do

on above code i want to pass path from a config file where i have many values.
so i want that my script should read each line of config file and match the pattern with stream and pass the path variable in my code.

Config file :

---------- Post updated at 06:44 AM ---------- Previous update was at 12:39 AM ----------

Requirement:

Config file:

key1 path1
key2 path2

search the key while reading the config file.
take the 2nd column of the that line as path
and pass it value during run time.

##Read 1st column as key of all the lines match the key with key inside the function and pass the 2nd column as argument during run time
#!/bin/sh
DATE=`date +"%Y-%m-%d %H:%M:%S"`
S1()
{
key=S1
counter=0
while [ $counter -le 100 ]
do
	for  files in `printf "%s\n" $path/*.RTM | head -3`
	do
		if [[ -f $files ]];
		then
		TT_FIRST=`awk -F ',' 'NR==1{print $1}' $files`
		TT_LAST=`awk -F ',' 'END{print $1}' $files`
		FILENAME=`ls $files | cut -d '/' -f6`
		TIMESTAMP=$( date -r $files +'%s')
		FILE_CR_TIME=$( date +'%Y-%m-%d %H:%M:%S' -r $files)
        TRANS_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +'%s')
		TRANS_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +'%s')
		RECORD_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +"%Y-%m-%d %H:%M:%S")
		RECORD_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +"%Y-%m-%d %H:%M:%S")
		TIME_LAG_FIRST=$(expr $TIMESTAMP - $TRANS_TIME_FIRST)
		TIME_LAG_LAST=$(expr $TIMESTAMP - $TRANS_TIME_LAST)
		DELAY_PROCESS=$((`expr $(date -u +"%s") - $TIMESTAMP`))
		TRANS_DIFFTIME=$(expr $TRANS_TIME_LAST - $TRANS_TIME_FIRST)
		counter=100
		else
			echo "file doesn't exist" >/dev/null
		fi	
		if [ "${FILENAME}" != "" ]; 
		then
			echo "${DATE} ${stream} ${FILENAME} ${FILE_CR_TIME} ${RECORD_TIME_FIRST} ${TIME_LAG_FIRST} ${RECORD_TIME_LAST} ${TIME_LAG_LAST} ${DELAY_PROCESS} ${TRANS_DIFFTIME}" | hdfs dfs -appendToFile - /bigpfstest/DPI_INVESTIG/AUDIT/FINAL.csv
		else
			echo "Filename is empty" >/dev/null 
		fi 
	done
((counter++))
sleep 5s
done
}


S1 $path &
4

as "config" files are named after the functions (or vice versa), you could use $FUNCNAME.

5

Why can't you simply grep the config file.

grep IUPS configfile |read key newpath
PATH=$newpath; export PATH
1 Like
6

did you try using a variable assigned from a pipe in /bin/sh ?

1 Like
7

My requirement is

--Read the config file line by line
--lines contains 8 columns
--validate from 8th column saying status if 1 active else inactive.
--1st column should pass as stream
--2nd column as path
--3rd column as counter
--4th column as counterlimit
--5th column from the files inside the path 1st column of the file and 4th column condition wise.

latency()
{
stream=$1
path=$2
counter=$3
counterlimit=$4
while [ $counter -le $counterlimit ]
do
	for  files in `printf "%s\n" $path/*bna_1* | head -3`
	do
		if [[ -f $files ]];
		then
		TT_FIRST=`awk -F ',' 'NR==1{print $1}' $files`
		TT_LAST=`awk -F ',' 'END{print $1}' $files`
		FILENAME=`ls $files | cut -d '/' -f6`
		TIMESTAMP=$( date -r $files +'%s')
		FILE_CR_TIME=$( date +'%Y-%m-%d %H:%M:%S' -r $files)
        TRANS_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +'%s')
		TRANS_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +'%s')
		RECORD_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +"%Y-%m-%d %H:%M:%S")
		RECORD_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +"%Y-%m-%d %H:%M:%S")
		TIME_LAG_FIRST=$(expr $TIMESTAMP - $TRANS_TIME_FIRST)
		TIME_LAG_LAST=$(expr $TIMESTAMP - $TRANS_TIME_LAST)
		DELAY_PROCESS=$((`expr $(date -u +"%s") - $TIMESTAMP`))
		TRANS_DIFFTIME=$(expr $TRANS_TIME_LAST - $TRANS_TIME_FIRST)
		counter=100
		else
			echo "file doesn't exist" >/dev/null
		fi	
		if [ "${FILENAME}" != "" ]; 
		then
			echo "${DATE} ${stream} ${FILENAME} ${FILE_CR_TIME} ${RECORD_TIME_FIRST} ${TIME_LAG_FIRST} ${RECORD_TIME_LAST} ${TIME_LAG_LAST} ${DELAY_PROCESS} ${TRANS_DIFFTIME}" | hdfs dfs -appendToFile - /bigpfstest/DPI_INVESTIG/AUDIT/FINAL.csv
		else
			echo "Filename is empty" >/dev/null 
		fi 
	done
((counter++))
sleep 10s
done
}


ignore echo part fro  or a while now.

---------- Post updated at 06:32 AM ---------- Previous update was at 06:25 AM ----------

Stream  	path   				 Counter   	Counterlimit 	 TransactionDateColumn   DateType             SleepValue      Status
S1      /streamslz/LZ/S1        	0    		100 			1      					1					10				1
IUCS    /streamslz/LZ/AnritsuIUCS   0           100      		4						2					5				1
IUPS    /streamslz/LZ/IUPS      	0           100      		1						1					5				1