Compare last 90 logs and print repeating lines with >20

Shell Programming and Scripting
1

*log files are in date order

sample logs...

ciscoresets_20120314
ciscoresets_20120313
ciscoresets_20120312
ciscoresets_20120311
ciscoresets_20120310

cat ciscoresets_20120314
SYDGRE04,10,9
SYDGRE04,10,10
SYDGRE04,10,11
SYDGRE04,10,12
SYDGRE04,10,13
SYDGRE04,10,14
SYDGRE04,10,15
NEPGRE01,10,1
NEPGRE01,10,2
NEPGRE01,10,3
NEPGRE01,10,4
NEPGRE01,10,5
NEPGRE01,10,6
NEPGRE01,10,7
NEPGRE01,10,8
NEPGRE01,10,9
NEPGRE01,10,10
NEPGRE01,10,11
NEPGRE01,10,12
NEPGRE01,10,13
NEPGRE01,10,14
NEPGRE01,10,15
NEPGRE01,10,16


cat ciscoresets_20120313
SYDGRE04,10,1
SYDGRE04,10,2
SYDGRE04,10,3
SYDGRE04,10,4
SYDGRE04,10,5
SYDGRE04,10,6
SYDGRE04,10,7
SYDGRE04,10,8
SYDGRE04,10,9
SYDGRE04,10,10
SYDGRE04,10,11
SYDGRE04,10,12
SYDGRE04,10,13
SYDGRE04,10,14
SYDGRE04,10,15
NEPGRE01,10,4
NEPGRE01,10,5
NEPGRE01,10,6
NEPGRE01,10,7
NEPGRE01,10,8
NEPGRE01,10,9
2

Try this (not tested)

cat `ls -1 ciscoresets_* | sort -r | head -90` | sort | uniq -c | awk '$1>20 {print $2}'
3

Hi, try this:

awk '
  function fdate( fd ) {
    sub(/^[^0-9]*/, nothing, fd ); return fd
  }
  function pattern( fd ) {
    sub(/[0-9]*$/, "*" , fd ); return fd
  }
  {
    MATCH[$1]=0
  }
  END {
    end=fdate(FILENAME)
    while ( "printf \"%s\n\" " pattern(FILENAME) | getline file ) {
      if ( fdate(file) >= start && fdate(file) < end ) list=list FS file
    }
    while( "cat "list | getline record ) {
      if ( record in MATCH ) MATCH[record]++
    }
    for (record in MATCH){
      if ( ! MATCH[record]  ) print "NEW:" record
      if ( MATCH[record] > treshold ) print record, MATCH[record]
    }
  }
' treshold=20 start=20111214 ciscoresets_20120314 | sort -k2