Compare a file with all others then print off data

Shell Programming and Scripting
1

my script so far

nawk -F, 'NR==FNR{a[$1OFS$2OFS$3]++;next} a[$1OFS$2OFS$3]{b[$1OFS$2OFS$3]++}
END{for(i in b){if(b-1){print i";\t\t"b}else{print "NEW:"i";\t\t1"} } }' OFS=, 20111228.csv *.csv | sort
NE:221478,SHELF:13,SLOT:4;              2
NE:221726,SHELF:8,SLOT:1;               2
NE:222318,SHELF:14,SLOT:1;              9
NE:222799,SHELF:5,SLOT:2;               2
NE:230478,SHELF:10,SLOT:6;              2
NE:252848,SHELF:3,SLOT:5;               3
NE:267909,SHELF:1,SLOT:6;               2
NE:267909,SHELF:4,SLOT:4;               4
NE:509033,SHELF:2,SLOT:1;               11
NE:509622,SHELF:12,SLOT:2;              2
NE:541469,SHELF:11,SLOT:6;              31
NE:565538,SHELF:7,SLOT:4;               15
NE:571528,SHELF:8,SLOT:2;               12
NE:593848,SHELF:3,SLOT:1;               3
NE:594728,SHELF:3,SLOT:5;               32
NE:597708,SHELF:12,SLOT:6;              2
NE:602306,SHELF:18,SLOT:6;              4
NE:800189,SHELF:8,SLOT:3;               3
NE:801048,SHELF:3,SLOT:1;               2
NE:825890,SHELF:5,SLOT:1;               2
NE:852991,SHELF:11,SLOT:4;              20
NE:877073,SHELF:3,SLOT:3;               6
NE:984177,SHELF:1,SLOT:4;               6
NEW:NE:220072,SHELF:6,SLOT:3;           1
NEW:NE:221514,SHELF:14,SLOT:1;          1
NEW:NE:230293,SHELF:6,SLOT:2;           1
NEW:NE:239160,SHELF:15,SLOT:4;          1
NEW:NE:509860,SHELF:8,SLOT:3;           1
NEW:NE:602306,SHELF:6,SLOT:3;           1
NEW:NE:801048,SHELF:6,SLOT:2;           1
NEW:NE:825890,SHELF:2,SLOT:4;           1
NEW:NE:826030,SHELF:8,SLOT:3;           1
NEW:NE:841068,SHELF:8,SLOT:4;           1
NEW:NE:854479,SHELF:6,SLOT:5;           1
NEW:NE:895435,SHELF:5,SLOT:5;           1
NEW:NE:993180,SHELF:3,SLOT:1;           1

#cat 20111228.csv
NE:221726,SHELF:8,SLOT:1,01:00:02,Wed Dec 28 2011
NE:801048,SHELF:3,SLOT:1,01:30:02,Wed Dec 28 2011
NE:841068,SHELF:8,SLOT:4,02:00:03,Wed Dec 28 2011
NE:593848,SHELF:3,SLOT:1,02:30:09,Wed Dec 28 2011
NE:801048,SHELF:6,SLOT:2,04:00:01,Wed Dec 28 2011
NE:252848,SHELF:3,SLOT:5,04:30:03,Wed Dec 28 2011
NE:852991,SHELF:11,SLOT:4,05:30:01,Wed Dec 28 2011
NE:509033,SHELF:2,SLOT:1,06:30:01,Wed Dec 28 2011
NE:509033,SHELF:2,SLOT:1,09:00:01,Wed Dec 28 2011
NE:222318,SHELF:14,SLOT:1,10:00:01,Wed Dec 28 2011
NE:267909,SHELF:4,SLOT:4,12:30:02,Wed Dec 28 2011
NE:800189,SHELF:8,SLOT:3,13:30:02,Wed Dec 28 2011
NE:509622,SHELF:12,SLOT:2,14:30:01,Wed Dec 28 2011
NE:852991,SHELF:11,SLOT:4,14:30:01,Wed Dec 28 2011
NE:602306,SHELF:6,SLOT:3,15:00:01,Wed Dec 28 2011
NE:597708,SHELF:12,SLOT:6,15:30:01,Wed Dec 28 2011
NE:239160,SHELF:15,SLOT:4,15:30:01,Wed Dec 28 2011
NE:800189,SHELF:8,SLOT:3,16:00:01,Wed Dec 28 2011
NE:825890,SHELF:5,SLOT:1,17:00:02,Wed Dec 28 2011
NE:571528,SHELF:8,SLOT:2,17:30:01,Wed Dec 28 2011
NE:594728,SHELF:3,SLOT:5,17:30:02,Wed Dec 28 2011
NE:565538,SHELF:7,SLOT:4,18:30:03,Wed Dec 28 2011
NE:825890,SHELF:2,SLOT:4,18:30:04,Wed Dec 28 2011
NE:220072,SHELF:6,SLOT:3,19:00:03,Wed Dec 28 2011
NE:826030,SHELF:8,SLOT:3,19:00:03,Wed Dec 28 2011
NE:895435,SHELF:5,SLOT:5,19:00:06,Wed Dec 28 2011
NE:222799,SHELF:5,SLOT:2,19:00:06,Wed Dec 28 2011
NE:267909,SHELF:1,SLOT:6,19:30:01,Wed Dec 28 2011
NE:800189,SHELF:8,SLOT:3,19:30:01,Wed Dec 28 2011
NE:984177,SHELF:1,SLOT:4,20:00:02,Wed Dec 28 2011
NE:230293,SHELF:6,SLOT:2,20:30:02,Wed Dec 28 2011
NE:602306,SHELF:18,SLOT:6,20:30:02,Wed Dec 28 2011
NE:541469,SHELF:11,SLOT:6,20:30:02,Wed Dec 28 2011
NE:854479,SHELF:6,SLOT:5,20:30:02,Wed Dec 28 2011
NE:825890,SHELF:5,SLOT:1,20:30:02,Wed Dec 28 2011
NE:221478,SHELF:13,SLOT:4,21:00:01,Wed Dec 28 2011
NE:252848,SHELF:3,SLOT:5,21:00:01,Wed Dec 28 2011
NE:230478,SHELF:10,SLOT:6,21:00:01,Wed Dec 28 2011
NE:509860,SHELF:8,SLOT:3,21:00:02,Wed Dec 28 2011
NE:565538,SHELF:7,SLOT:4,21:30:02,Wed Dec 28 2011
NE:993180,SHELF:3,SLOT:1,22:00:01,Wed Dec 28 2011
NE:852991,SHELF:11,SLOT:4,22:00:02,Wed Dec 28 2011
NE:221514,SHELF:14,SLOT:1,22:00:02,Wed Dec 28 2011
NE:267909,SHELF:4,SLOT:4,22:30:01,Wed Dec 28 2011
NE:877073,SHELF:3,SLOT:3,23:00:02,Wed Dec 28 2011

see existing script in first output, current script is designed to pick up line highlighted in red as a repeat but because it has not been in previous files it should therefore be a NEW with a 2 count, if you understand?

:wall:

2

Hi llcooljatt,

I'm afraid that you will have to give us more details.

I've tested your script but output is different, did you miss any step?

$ awk -F, 'NR==FNR{a[$1OFS$2OFS$3]++;next} a[$1OFS$2OFS$3]{b[$1OFS$2OFS$3]++} END{for(i in b){if(b-1){print i";\t\t"b}else{print "NEW:"i";\t\t1"} } }' OFS=, 20111228.csv *.csv | sort
NE:252848,SHELF:3,SLOT:5;               2
NE:267909,SHELF:4,SLOT:4;               2
NE:509033,SHELF:2,SLOT:1;               2
NE:565538,SHELF:7,SLOT:4;               2
NE:800189,SHELF:8,SLOT:3;               3
NE:825890,SHELF:5,SLOT:1;               2
NE:852991,SHELF:11,SLOT:4;              3
NEW:NE:220072,SHELF:6,SLOT:3;           1
NEW:NE:221478,SHELF:13,SLOT:4;          1
NEW:NE:221514,SHELF:14,SLOT:1;          1
NEW:NE:221726,SHELF:8,SLOT:1;           1
NEW:NE:222318,SHELF:14,SLOT:1;          1
NEW:NE:222799,SHELF:5,SLOT:2;           1
NEW:NE:230293,SHELF:6,SLOT:2;           1
NEW:NE:230478,SHELF:10,SLOT:6;          1
NEW:NE:239160,SHELF:15,SLOT:4;          1
NEW:NE:267909,SHELF:1,SLOT:6;           1
NEW:NE:509622,SHELF:12,SLOT:2;          1
NEW:NE:509860,SHELF:8,SLOT:3;           1
NEW:NE:541469,SHELF:11,SLOT:6;          1
NEW:NE:571528,SHELF:8,SLOT:2;           1
NEW:NE:593848,SHELF:3,SLOT:1;           1
NEW:NE:594728,SHELF:3,SLOT:5;           1
NEW:NE:597708,SHELF:12,SLOT:6;          1
NEW:NE:602306,SHELF:18,SLOT:6;          1
NEW:NE:602306,SHELF:6,SLOT:3;           1
NEW:NE:801048,SHELF:3,SLOT:1;           1
NEW:NE:801048,SHELF:6,SLOT:2;           1
NEW:NE:825890,SHELF:2,SLOT:4;           1
NEW:NE:826030,SHELF:8,SLOT:3;           1
NEW:NE:841068,SHELF:8,SLOT:4;           1
NEW:NE:854479,SHELF:6,SLOT:5;           1
NEW:NE:877073,SHELF:3,SLOT:3;           1
NEW:NE:895435,SHELF:5,SLOT:5;           1
NEW:NE:984177,SHELF:1,SLOT:4;           1
NEW:NE:993180,SHELF:3,SLOT:1;           1

Regards,
Birei

3

the output is different as I have many of *.csv files, you will not have the same

4

What does this mean?

Regards,
Birei

1 Like
5

NE:825890,SHELF:5,SLOT:1 has not been in *.csv

6

I don't like much your approach to the problem. It is difficult to understand, and more if I can't test it.

This is my shot in the dark:

nawk -F, '
NR==FNR {
    a[$1OFS$2OFS$3]++;
    next
}

NR != FNR {
    b[$1OFS$2OFS$3]++;
}

END {
    for ( key in a ) {
        if ( b[key] ) {
            b[key]++
        } else {
            print "NEW: " key ";\t\t" a[key]
        }
    }

    for ( key in b ) {
        if ( b[key] == 1 ) {
            print "NEW: " key ";\t\t" b[key]
        } else {
            print key ";\t\t" b[key]
        }
    }
}' OFS=, 20111228.csv *.csv | sort

Regards,
Birei

1 Like
7

I am sorry I am new to programming hence not the best at understanding.

Have you got line output please?

8

When you run that piece of code, what is the result? Any error?

Regards,
Birei

1 Like
9

can you provide me with the line i.e

nawk -F, 'NR==FNR{a[$1OFS$2OFS$3]++;next} a[$1OFS$2OFS$3]{b[$1OFS$2OFS$3]++}
END{for(i in b){if(b-1){print i";\t\t"b}else{print "NEW:"i";\t\t1"} } }' OFS=, 20111228.csv *.csv | sort

---------- Post updated at 06:11 PM ---------- Previous update was at 06:03 PM ----------

this prints everything as new and 1 multiple

NEW: NE:978303,SHELF:9,SLOT:1;          1
NEW: NE:978303,SHELF:9,SLOT:3;          1
NEW: NE:978303,SHELF:9,SLOT:4;          1
NEW: NE:984177,SHELF:2,SLOT:3;          1
NEW: NE:984177,SHELF:4,SLOT:2;          1
NEW: NE:984966,SHELF:3,SLOT:1;          1
NEW: NE:984966,SHELF:3,SLOT:3;          1
NEW: NE:985360,SHELF:2,SLOT:1;          1
NEW: NE:985360,SHELF:2,SLOT:4;          1
NEW: NE:985360,SHELF:3,SLOT:3;          1
NEW: NE:985360,SHELF:3,SLOT:5;          1
NEW: NE:985360,SHELF:4,SLOT:3;          1
NEW: NE:986089,SHELF:2,SLOT:4;          1
NEW: NE:986089,SHELF:3,SLOT:2;          1
NEW: NE:986089,SHELF:5,SLOT:3;          1
NEW: NE:986089,SHELF:6,SLOT:1;          1
NEW: NE:986089,SHELF:6,SLOT:5;          1
NEW: NE:990237,SHELF:2,SLOT:4;          1
NEW: NE:990237,SHELF:4,SLOT:3;          1
NEW: NE:990237,SHELF:7,SLOT:6;          1
NEW: NE:992658,SHELF:2,SLOT:3;          1
NEW: NE:992658,SHELF:2,SLOT:5;          1
NEW: NE:992658,SHELF:4,SLOT:2;          1
NEW: NE:992658,SHELF:5,SLOT:3;          1
NEW: NE:992711,SHELF:1,SLOT:4;          1
NEW: NE:992711,SHELF:2,SLOT:1;          1
NEW: NE:993180,SHELF:2,SLOT:4;          1
NEW: NE:993180,SHELF:3,SLOT:5;          1
NEW: NE:993180,SHELF:5,SLOT:4;          1
NEW: NE:993180,SHELF:5,SLOT:6;          1
NEW: NE:993880,SHELF:1,SLOT:4;          1
10

maybe you can try like this [ as far as i understand what you want ]

# awk -F, 'NR==FNR{a[$1OFS$2OFS$3]++;next}{b[$1OFS$2OFS$3]++}
END{for(i in a){for(j in b){if(i==j){cc=a+b[j]}else{cn++};
if(cn==length(b)){print "NEW ENTRY FOUND -->",i,a;w=0}};if(w!=0){print i,cc};w=1;cn=0}}
' new old*

regards
ygemici

1 Like
11

Hi I will check when I'm in work, can you check my previous posts, I had help from initial script from ahamed but just need it to do one more thing i.e increment NEW resets it has found i.e not look for -1 but give correct figure as NEW if the line is not in *.csv

---------- Post updated 30-12-11 at 07:21 AM ---------- Previous update was 29-12-11 at 09:22 PM ----------

#nawk -F, 'NR==FNR{a[$1OFS$2OFS$3]++;next}{b[$1OFS$2OFS$3]++}
> END{for(i in a){for(j in b){if(i==j){cc=a[i]+b[j]}else{cn++};
> if(cn==length(b)){print "NEW ENTRY FOUND -->",i,a[i];w=0}};if(w!=0){print i,cc};w=1;cn=0}}
> ' 20111228.csv *.csv
nawk: can't read value of b; it's an array name.
input record number 45, file 20111228.csv
source line number 3

12

then try like this

# nawk -F, 'NR==FNR{a[$1OFS$2OFS$3]++;next}{b[$1OFS$2OFS$3]++}
END{for(j in b){bx++};for(i in a){for(j in b){if(i==j){cc=a+b[j]}else{cn++}
if(cn==bx){print "NEW ENTRY FOUND -->",i,a;w=0}};if(w!=0){print i,cc};w=1;cn=0}}
' 20111228.csv *.csv
1 Like
13

your script

nawk -F, 'NR==FNR{a[$1OFS$2OFS$3]++;next}{b[$1OFS$2OFS$3]++}
> END{for(j in b){bx++};for(i in a){for(j in b){if(i==j){cc=a+b[j]}else{cn++}
> if(cn==bx){print "NEW ENTRY FOUND -->",i,a;w=0}};if(w!=0){print i,cc};w=1;cn=0}}
> ' failed_lcss_reboots_20111229.csv *.csv
NE:565538 SHELF:10 SLOT:4 5
NE:851608 SHELF:4 SLOT:2 12
NE:504024 SHELF:4 SLOT:2 2
NE:571528 SHELF:1 SLOT:5 3
NE:236260 SHELF:15 SLOT:5 2
NE:220172 SHELF:3 SLOT:4 4
NE:239160 SHELF:11 SLOT:5 18
NE:221478 SHELF:3 SLOT:2 4
NE:565538 SHELF:7 SLOT:4 19
NE:883948 SHELF:10 SLOT:2 2
NE:564127 SHELF:8 SLOT:4 3
NE:573910 SHELF:5 SLOT:4 2
NE:602312 SHELF:3 SLOT:2 5
NE:972774 SHELF:5 SLOT:2 2
NE:993180 SHELF:2 SLOT:6 5
NE:978303 SHELF:3 SLOT:2 3
NE:221478 SHELF:10 SLOT:4 10
NE:571452 SHELF:11 SLOT:5 6
NE:903793 SHELF:3 SLOT:2 3
NE:239337 SHELF:8 SLOT:1 4
NE:852991 SHELF:11 SLOT:4 24
NE:222372 SHELF:5 SLOT:4 3
NE:267909 SHELF:4 SLOT:4 6
NE:858436 SHELF:6 SLOT:5 4
NE:220251 SHELF:7 SLOT:4 6
NE:509622 SHELF:12 SLOT:2 4
NE:238974 SHELF:12 SLOT:4 3
NE:236313 SHELF:16 SLOT:3 7
NE:851568 SHELF:5 SLOT:6 6
NE:554105 SHELF:5 SLOT:5 8
NE:221768 SHELF:4 SLOT:6 2
NE:571528 SHELF:12 SLOT:5 5
NE:828273 SHELF:2 SLOT:2 9
NE:575322 SHELF:8 SLOT:1 4
NE:565769 SHELF:5 SLOT:2 3
NE:597708 SHELF:11 SLOT:5 2
NE:509033 SHELF:2 SLOT:1 27
NE:220012 SHELF:5 SLOT:5 6
NE:220151 SHELF:6 SLOT:1 2
NE:829608 SHELF:1 SLOT:6 3

previous script

nawk -F, 'NR==FNR{a[$1OFS$2OFS$3]++;next} a[$1OFS$2OFS$3]{b[$1OFS$2OFS$3]++}
END{for(i in b){if(b-1){print i";\t\t"b}else{print "NEW:"i";\t\t1"} } }' OFS=, 20111229.csv *.csv | sort
NE:220012,SHELF:5,SLOT:5;               3
NE:220172,SHELF:3,SLOT:4;               3
NE:220251,SHELF:7,SLOT:4;               5
NE:221478,SHELF:10,SLOT:4;              7
NE:221478,SHELF:3,SLOT:2;               3
NE:222372,SHELF:5,SLOT:4;               2
NE:236313,SHELF:16,SLOT:3;              4
NE:238974,SHELF:12,SLOT:4;              2
NE:239160,SHELF:11,SLOT:5;              17
NE:239337,SHELF:8,SLOT:1;               3
NE:267909,SHELF:4,SLOT:4;               5
NE:509033,SHELF:2,SLOT:1;               20
NE:509622,SHELF:12,SLOT:2;              3
NE:554105,SHELF:5,SLOT:5;               7
NE:564127,SHELF:8,SLOT:4;               2
NE:565538,SHELF:10,SLOT:4;              3
NE:565538,SHELF:7,SLOT:4;               17
NE:565769,SHELF:5,SLOT:2;               2
NE:571452,SHELF:11,SLOT:5;              5
NE:571528,SHELF:1,SLOT:5;               2
NE:571528,SHELF:12,SLOT:5;              4
NE:575322,SHELF:8,SLOT:1;               3
NE:602312,SHELF:3,SLOT:2;               3
NE:828273,SHELF:2,SLOT:2;               8
NE:829608,SHELF:1,SLOT:6;               2
NE:851568,SHELF:5,SLOT:6;               5
NE:851608,SHELF:4,SLOT:2;               11
NE:852991,SHELF:11,SLOT:4;              21
NE:858436,SHELF:6,SLOT:5;               3
NE:903793,SHELF:3,SLOT:2;               2
NE:978303,SHELF:3,SLOT:2;               2
NE:993180,SHELF:2,SLOT:6;               4
NEW:NE:220151,SHELF:6,SLOT:1;           1
NEW:NE:221768,SHELF:4,SLOT:6;           1
NEW:NE:223142,SHELF:11,SLOT:5;          1
NEW:NE:236260,SHELF:15,SLOT:5;          1
NEW:NE:504024,SHELF:4,SLOT:2;           1
NEW:NE:573910,SHELF:5,SLOT:4;           1
NEW:NE:597708,SHELF:11,SLOT:5;          1
NEW:NE:883948,SHELF:10,SLOT:2;          1
NEW:NE:972774,SHELF:5,SLOT:2;           1

your script seems to add an extra 1 to the figures, where as my old one is pretty much there only thing I need it to do is not look for -1 for new resets when comparing 20111229.csv with *.csv but give accurate reflection i.e instead of 1 this could be 10

i.e

NEW:NE:883948,SHELF:10,SLOT:2; 10

highlight as NEW on left

14 
nawk -F, 'NR==FNR{a[$1OFS$2OFS$3]++;next} a[$1OFS$2OFS$3]{b[$1OFS$2OFS$3]++}
END{for(i in b){if(b-1&&a!=b){print i";\t\t"b}else{print "NEW:"i";\t\t"b} } }' OFS=, 20111229.csv *.csv

--ahamed

1 Like
15

ahamed I am grateful excellent work !

can you explain the modified code so that I understand better

specifically

{if(b-1&&a!=b){print i";\t\t"b}else{print "NEW:"i";\t\t"b}
16

[quote=llcooljatt;302586048]
your script

nawk -F, 'NR==FNR{a[$1OFS$2OFS$3]++;next}{b[$1OFS$2OFS$3]++}
> END{for(j in b){bx++};for(i in a){for(j in b){if(i==j){cc=a+b[j]}else{cn++}
> if(cn==bx){print "NEW ENTRY FOUND -->",i,a;w=0}};if(w!=0){print i,cc};w=1;cn=0}}
> ' failed_lcss_reboots_20111229.csv *.csv

your script seems to add an extra 1 to the figures, where as my old one is pretty much there only thing I need it to do is not look for -1 for new resets when comparing 20111229.csv with *.csv but give accurate reflection i.e instead of 1 this could be 10

i.e

NEW:NE:883948,SHELF:10,SLOT:2;          10

highlight as NEW on left

i don't understand exactly what you want but i guess output should not contain counts which in the new file about old records..

for example
# cat old*
NE:221726,SHELF:8,SLOT:1,01:00:02,Wed Dec 28 2011
NE:801048,SHELF:3,SLOT:1,01:30:02,Wed Dec 28 2011
NE:841068,SHELF:8,SLOT:4,02:00:03,Wed Dec 28 2011
NE:593848,SHELF:3,SLOT:1,02:30:09,Wed Dec 28 2011
NE:593848,SHELF:3,SLOT:1,02:30:09,Wed Dec 28 2011
NE:801048,SHELF:6,SLOT:2,04:00:01,Wed Dec 28 2011
NE:801048,SHELF:6,SLOT:2,04:00:01,Wed Dec 28 2011
NE:221726,SHELF:8,SLOT:1,01:00:02,Wed Dec 28 2011
NE:221726,SHELF:8,SLOT:1,01:00:02,Wed Dec 28 2011
NE:221726,SHELF:8,SLOT:1,01:00:02,Wed Dec 28 2011
NE:221726,SHELF:8,SLOT:1,01:00:02,Wed Dec 28 2011
NE:801048,SHELF:3,SLOT:1,01:30:02,Wed Dec 28 2011
NE:801048,SHELF:3,SLOT:1,01:30:02,Wed Dec 28 2011
NE:801048,SHELF:3,SLOT:1,01:30:02,Wed Dec 28 2011
NE:801048,SHELF:3,SLOT:1,01:30:02,Wed Dec 28 2011
NE:841068,SHELF:8,SLOT:4,02:00:03,Wed Dec 28 2011
NE:593848,SHELF:3,SLOT:1,02:30:09,Wed Dec 28 2011
NE:801048,SHELF:6,SLOT:2,04:00:01,Wed Dec 28 2011
NE:801048,SHELF:6,SLOT:2,04:00:01,Wed Dec 28 2011
NE:801048,SHELF:6,SLOT:2,04:00:01,Wed Dec 28 2011
NE:801048,SHELF:6,SLOT:2,04:00:01,Wed Dec 28 2011

# cat new
NE:888888,SHELF:3,SLOT:1,01:30:02,Wed Dec 28 2011
NE:111111,SHELF:3,SLOT:1,01:30:02,Wed Dec 28 2011
NE:221726,SHELF:8,SLOT:1,01:00:02,Wed Dec 28 2011
NE:888888,SHELF:3,SLOT:1,01:30:02,Wed Dec 28 2011
NE:801048,SHELF:3,SLOT:1,01:30:02,Wed Dec 28 2011
NE:801048,SHELF:3,SLOT:1,01:30:02,Wed Dec 28 2011
NE:841068,SHELF:8,SLOT:4,02:00:03,Wed Dec 28 2011
NE:593848,SHELF:3,SLOT:1,02:30:09,Wed Dec 28 2011
NE:801048,SHELF:6,SLOT:2,04:00:01,Wed Dec 28 2011
NE:801048,SHELF:6,SLOT:2,04:00:01,Wed Dec 28 2011

i removed record counts from new file and i add a condition for recurrence records.

# nawk -F, 'NR==FNR{a[$1FS$2FS$3]++;next}{b[$1FS$2FS$3]++}
END{for(j in b){bx++};for(i in a){for(j in b){if(i==j){cc=b[j]}else{cn++}
if(cn==bx){print "NEW->",i";\t\t",a;}w=0};if(w=!0&&cn<bx){;print i";\t\t",cc};w=1;cn=0}}
' new old*
NE:801048,SHELF:3,SLOT:1;                5
NE:593848,SHELF:3,SLOT:1;                3
NEW-> NE:888888,SHELF:3,SLOT:1;          2
NE:801048,SHELF:6,SLOT:2;                6
NEW-> NE:111111,SHELF:3,SLOT:1;          1
NE:841068,SHELF:8,SLOT:4;                2
NE:221726,SHELF:8,SLOT:1;                5

@ahamed101 code gives same result but new records..you can use this code when you want just old records..

# nawk -F, 'NR==FNR{a[$1OFS$2OFS$3]++;next} a[$1OFS$2OFS$3]{b[$1OFS$2OFS$3]++}
> END{for(i in b){if(b-1&&a!=b){print i";\t\t"b}else{print "NEW:"i";\t\t"b} } }' OFS=, new old*
NE:801048,SHELF:3,SLOT:1;               5
NE:593848,SHELF:3,SLOT:1;               3
NE:801048,SHELF:6,SLOT:2;               6
NE:841068,SHELF:8,SLOT:4;               2
NE:221726,SHELF:8,SLOT:1;               5

regards
ygemici

1 Like
17

can you explain or write comments for your code please, I am new to this and any help from you guys is appreciated.

18

nawk -F, ## determine the our FS=,

'NR==FNR{a[$1FS$2FS$3]++;next} ## execute this until the NR equal to FNR
so NR means  number of input records and it will be increase as long as tha read new records from all input files..
FNR means current record number in the current file so FNR holds the number of record for each new file as separately.
when the awk has started to execute for input files(from stdinput or pipe), NR and FNR equals 1
and both of them increases synchronous while started to read a new file..
when a new file is read from awk then NR will continue to increase but FNR reset to zero for every new file at each time.
so NR and FNR is same while a new file processing and so NR and FNR is eqaul for first file..

in that case, ...

NR==FNR{a[$1FS$2FS$3]++;next} ## process when NR==FNR (so execute for first file which read ) and then
for example our line is "NE:888888,SHELF:3,SLOT:1,01:30:02,Wed Dec 28 2011"
a[$1FS$2FS$3]++  ## $1 "NE:888888" FS=, and $2 "SHELF:3" and FS=, $3 "SLOT:1" and assing to array(a) 
a[NE:888888,SHELF:3,SLOT:1] --> our first index (is a string) array(a) holds our indexes (it is an associative array)
read goes on new records from all first file..
for exa read let's same record so new line is "NE:888888,SHELF:3,SLOT:1,01:30:02,Wed Dec 28 2011" ( same the above)
a[$1FS$2FS$3]++  ## 
increments the count for our index (NE:888888,SHELF:3,SLOT:1) so "print a[$1FS$2FS$3]" gives the count for this lines 
finally all lines accumulated in array indexes with counts

.....

next ## go to next record(line). hereby awk is forced to read the next record
NR==FNR{....;next} ## so read all lines in first file.
When it comes to the end of first file so awk process next file(s)

{b[$1FS$2FS$3]++} ## in the same way accumulates all lines from other file(s)

END ## after all the files has been read then execute this code that in the END{...}
{for(j in b){bx++}; ## we find array length and assing to "bx" value (bx increase while read an index val in b array)

for(i in a){  ## read indexes in a array (i --> index ,, a --> array name)
for(j in b){  ## same...
if(i==j){cc=b[j]} ## if "NE:888888,SHELF:3,SLOT:1" equals to "NE:221726,SHELF:8,SLOT:1" (for exa) then "cc=b[j]" [index count]

else{cn++} ## else increase value the "cn" for how many indexes have for not equal to "NE:888888,SHELF:3,SLOT:1"
## so this comparing will continue while "for-loop" for b array indexes [ remember b array indexes come from old files not first]

if(cn==bx){print "NEW->",i";\t\t",a;}w=0} ## if cn equals to bx (so it's means that didnt find any matches between new file and other files indexes)
## set w to zero (for do not write as OLD value)
if then write as NEW value and a (count for this index val)

if(w=!0&&cn<bx){;print i";\t\t",cc};w=1;cn=0}} ## if w val is non-zero and cn smaller than the bx val
## if w is differ from zero it means NEW value has not been found already (above) 
## and bx(length of b array is greater than cn (so there is a matching record)
## (remember these(i) values holds from new file indexes)
..actually it could have been as "-1" ) ## i modified like way for my code for this
## then can write as OLD value
## so write as OLD value and cc (with count for this(j) index which from other files (so b array))
' new old*  ## new file and old files

# nawk -F, 'NR==FNR{a[$1FS$2FS$3]++;next}{b[$1FS$2FS$3]++}
END{for(j in b){bx++};for(i in a){for(j in b){if(i==j){cc=b[j]}else{cn++}
if(cn==bx){print "NEW->",i";\t\t",a;}w=0};if(w=!0&&cn==bx-1){;print i";\t\t",cc};w=1;cn=0}}
' new old*

regards
ygemici

1 Like
19

@ygemici : the file arguments which is fed to awk is to be noted. 20111229.csv *.csv

20111229.csv is fed in twice and hence it takes care of both new and old with the logic.

--ahamed

1 Like
20

@ahamed can I say thanks for this code it will save me hours of manual work, 1 thing can you explain

if(b-1&&a!=b)

:b::D:b:

ok think iv worked it out, if my b array index is equal to -1 && (boolean) array a index is not = b index i.e no lines match in array b then {print "NEW:"i";\t\t"b[i]} is that a fair assessment ?