Hey Guys,
Does anyone know how to count the separate amount of words in a text file?
e.g
the 5
and 20
Furthermore does anyone know how to convert whole numbers in decimals?
Thanks
tr '[ \r\t]' ' ' < file | wc -l
For the second I don't understand the question. what do you get in and what do you want out?
Hey,
Thanks for the post
I want to convert the number you'd get for each word counted e.g. the: 46, into a decimal number so in this case '46'.
Thanks,
Try this
awk '{for (i=1;i<=NF;i++){a[$i]++}} END{for (j in a){print j,a[j]}}' <inputFile>
Explain what you're trying to achieve and post your input file and the desired output.
xargs -n1 <infile | wc -l
---------- Post updated at 07:47 PM ---------- Previous update was at 07:32 PM ----------
For fun ... with pure sed :
# cat tst
1 2 3 4
A B C
XX YY ZZ AA WWW
# ./ts2 <tst
12
For those who don't already know this example ...
@Bartus (here is a geek example, i hope you'll especially enjoy it !!! :D)
HERE IS THE CHALLENGE :
Understand how this sed code works (this should make your brain work quite a bit ) 
!!!
(i mean the whole algoritm, not only what each line does)
# cat ts2
#!/bin/sed -nf
s/[[:blank:]][[:blank:]]*/ /g
s/^/ /
s/ [^ ][^ ]*/a /g
s/ //g
H
x
s/\n//
/aaaaaaaaaa/! bx; s/aaaaaaaaaa/b/g
/bbbbbbbbbb/! bx; s/bbbbbbbbbb/c/g
/cccccccccc/! bx; s/cccccccccc/d/g
/dddddddddd/! bx; s/dddddddddd/e/g
/eeeeeeeeee/! bx; s/eeeeeeeeee/f/g
/ffffffffff/! bx; s/ffffffffff/g/g
/gggggggggg/! bx; s/gggggggggg/h/g
s/hhhhhhhhhh//g
:x
$! { h; b; }
:y
/a/! s/[b-h]*/&0/
s/aaaaaaaaa/9/
s/aaaaaaaa/8/
s/aaaaaaa/7/
s/aaaaaa/6/
s/aaaaa/5/
s/aaaa/4/
s/aaa/3/
s/aa/2/
s/a/1/
y/bcdefgh/abcdefg/
/[a-h]/ by
p
Hey Guys
I recently posted yesterday about trying to count the amount of separate words that exists in a text file e.g. walle.txt.
i want the output to give to give me a list of words with a number next indicating how many times its came up in the file e.g:
cat 20
the 11
if 40
I'm completely new to Unix, I'm currently using the bash terminal from a Macbook Pro. I am running this on some example file scripts, is what i'm trying to do possible? if so please help.
Thanks
How the posted solutions NOT fulfilling the above request?
As always, please provide a sample input file a desired output (using code tags).
xargs -n1 <yourfile.txt | sort | uniq -c
---------- Post updated at 05:59 PM ---------- Previous update was at 05:54 PM ----------
the simple quote may annoy xargs so we can change it into a white space at first with a sed statement :
sed "s/'/ /g" yourfile.txt | xargs -n1 | sort | uniq -c
Hey,
Thanks for the reply, i just tried both statements and 'illegal argument count' keeps coming up :\
Please upload your text file so we can try with a real example and we will let you know if some other step are needed
Try to go through a tempfile then.
give a try to this:
sed "s/'/ /g" yourfile.txt >file.tmp
xargs -n1 <file.tmp >file.tmp.2
sort file.tmp.2 >file.tmp.sorted
uniq -c file.tmp.sorted >wordcount.txt
rm file.tmp*
cat wordcount.txt
Okay Thanks,
I've attached it below
OMG ... i doesn't really look like a txt file, but more like a binary file !
(... forget my previous post)
give a try to this:
strings yourfile.txt | sed "s/'/ /g" | xargs -n1 | sort | uniq -c
I gave it a go and it still says 'illegal argument count' 
In fact your document is a Microsoft Word document
Open it in word, and save it as a txt ...
# file walle.txt
walle.txt: Microsoft Word Document
You must change case to lower, I like tr, then separate all the words, say using sed, then count them, sort to uniq -c.
tr '[:upper:]' '[:lower]' <yourfile.txt | sed '
s/[^a-z]\{1,99\}/\
/g
' | sort | uniq -c >wordcount.txt
For big files with mostly repeating words, you may need a less disk-space intensive, intelligent solution. I wrote this in C, instructions included, replace "sort | uniq -c" with "aggsx -l". (Some features are aimed at the Interbase/Firebird RDBMS.) (You could write a more elegant and faster on using a hash map container in C++ or JAVA, and do the case and separation in code, too.)
$ cat mysrc/aggsx.c
#include <stdio.h>
#include <limits.h>
#include <errno.h>
#include <stdlib.h>
#include <strings.h>
static long double sum = 0.0 ;
static long double nval ;
static unsigned long lct = 0 ;
static unsigned long nct = 0 ;
static unsigned long ll2 ;
static unsigned long nvc = 0 ;
static unsigned long dct = 0 ;
static unsigned long act = 0 ;
static unsigned long ll ;
static unsigned long mpc = 0 ;
static unsigned long *vct = NULL ; /* value counts */
static unsigned long *lp ;
static char **vl = NULL ; /* value list */
static char **cpp ;
static char *cp ;
static char *cp2 ;
static char *cp3 ;
static char *me = "" ;
static char *mp = "" ;
static char *pfx = NULL ;
static int i ;
static int d = 0 ; /* -d option */
static int u = 0 ; /* -u option */
static int l = 0 ; /* -l option */
static int b = 0 ; /* -b option */
static int num = 1 ;
static int lfm ; /* line feed missing */
static char buf[66000] ;
static void fmv( char *val )
{
unsigned long cv ;
unsigned long cl = 0 ;
unsigned long ch ;
int r ;
char **cf ;
char **ct ;
char **ce ;
unsigned long *lf ;
unsigned long *lt ;
if ( dct )
for ( cl = 0, ch = dct - 1 ; cl <= ch ; )
{
cv = ( ch + cl ) >> 1 ;
r = strcmp( val, vl[cv] );
if ( r > 0 )
{
cl = cv + 1 ;
}
else if ( r < 0 )
{
if ( cv )
ch = cv - 1 ;
else
break ;
}
else
{
lt = vct + cv ;
*lt += 1 ;
if ( d
&& *lt == 2 ) /* report dups */
{
if ( 0 > printf( "%s\n", val )
|| fflush( stdout ) )
{
if ( ferror( stdout ) )
{
perror( "stdout" );
exit( 1 );
}
exit( 0 );
}
}
return ;
}
}
if ( u ) /* report unique */
{
if ( 0 > printf( "%s\n", val )
|| fflush( stdout ) )
{
if ( ferror( stdout ) )
{
perror( "stdout" );
exit( 1 );
}
exit( 0 );
}
}
cv = dct ;
if ( ++dct > act )
{
act += 1024 ;
if ( !( vl = realloc( vl, act * sizeof( char* ) ) ) )
{
perror( "realloc()" );
exit( 1 );
}
if ( !( vct = realloc( vct, act * sizeof( long ) ) ) )
{
perror( "realloc()" );
exit( 1 );
}
}
for ( ce = vl + cl,
cf = ( ( ct = vl + cv ) - 1 ),
lf = ( ( lt = vct + cv ) - 1 ) ;
ct > ce ;
cf--, ct--, lf--, lt-- )
{
*ct = *cf ;
*lt = *lf ;
}
*lt = 1 ;
if ( !( *ct = malloc( strlen( val ) + 1 ) ) )
{
perror( "malloc()" );
exit( 1 );
}
strcpy( *ct, val );
return ;
}
int main( int argc, char **argv ){
setvbuf( stdin, NULL, _IOFBF, PIPE_MAX );
setvbuf( stdout, NULL, _IOFBF, PIPE_MAX );
for ( i = 1 ; i < argc ; i++ )
{
if ( !strcmp( argv[1], "-b" ) )
{
b = 1 ;
continue ;
}
if ( !strcmp( argv[1], "-l" ) )
{
l = 1 ;
continue ;
}
if ( !strcmp( argv[1], "-p" )
&& ++i < argc )
{
pfx = argv;
continue ;
}
if ( !strcmp( argv[1], "-u" ) )
{
u = 1 ;
continue ;
}
if ( !strcmp( argv[1], "-d" ) )
{
d = 1 ;
continue ;
}
if ( !strcmp( argv[1], "-h" ) )
{
fputs(
"CtD|CtN|Min|CtMin|Max|CtMax|Avg|Med|MPop|CtMPop|Ct\n",
stdout );
continue ;
}
fputs(
"Usage:\n"
"\n"
"aggsx [ -b ] [ -l ] [ -p <prefix> ] [ -u ] [ -d ] [ -h ]\n"
"\n"
"Computes the count distinct, count null, min, count of min, max,\n"
"count of max, average (mean) of not null values if numeric,\n"
"median of not null values, largest of the most popular values,\n"
"count of that most popular value.\n"
"\n"
"If -l is present, first prints out all values in order and their counts,\n"
"null last, but no aggregates.\n"
"If -b is present, prints out like -l and then prints aggregates.\n"
"If -p is present, the aggregate is prefixed with '<prefix>|'.\n"
"If -u is present, just immediately prints out unique values.\n"
"If -d is present, just immediately prints out duplicated values.\n"
"If -h is present, prefixes values line with header line:\n"
"CtD|CtN|Min|CtMin|Max|CtMax|Avg|Med|MPop|CtMPop\n"
"\n" , stderr );
exit( 1 );
}
while( fgets( buf, sizeof( buf ), stdin ) )
{
lct++ ;
for ( cp = buf, cp2 = cp3 = NULL, lfm = 1 ; *cp ; cp++ )
{
switch( *cp )
{
case '\n':
lfm = 0 ;
/* intentional fall through */
case '\r':
/* intentional fall through */
case ' ':
/* intentional fall through */
case '\t':
continue ;
/* intentional fall through */
default:
if ( !cp2 )
{
cp2 = cp ;
}
cp3 = cp ;
}
}
if ( lfm )
{
fprintf( stderr, "\nFatal: Data line %lu too long!\n",
lct );
exit( 1 );
}
if ( cp3 )
{
*(++cp3) = NULL ;
cp = cp2 ;
}
if ( strcmp( cp, "<null>" ) )
{
fmv( cp );
}
else
{
nct++ ;
}
}
if ( ferror( stdin ) )
{
perror( "stdin" );
exit( 1 );
}
if ( u
|| d )
exit( 0 );
if ( l || b )
{
for ( ll = 0, cpp = vl, lp = vct ;
ll < dct ;
ll++, lp++, cpp++ )
{
if ( 0 > printf( "%lu\t%s\n", *lp, *cpp ) )
{
if ( ferror( stdout ) )
{
perror( "stdout" );
exit( 1 );
}
exit( 0 );
}
}
if ( nct
&& 0 > printf( "%lu\t%s\n", nct, "<null>" ) )
{
if ( ferror( stdout ) )
{
perror( "stdout" );
exit( 1 );
}
exit( 0 );
}
if ( !b )
{
exit( 0 );
}
}
for ( ll = 0L, cpp = vl, lp = vct, ll2 = ( ( lct - nct ) >> 1 ) + nct ;
ll < dct ; ll++, lp++, cpp++ )
{
cp = *cpp ;
if ( *lp >= mpc )
{
mpc = *lp ;
mp = cp ;
}
if ( *cp
&& num )
{
errno = 0 ;
nval = strtod( cp, &cp2 );
if ( errno /* underflow or overflow */
|| ( cp2 == cp ) /* didn't like the characters */
|| *cp2 ) /* didn't like some */
{
num = 0 ;
}
else
{
sum += ( nval * *lp ) ;
nvc += *lp ;
}
}
if ( ll2 <= lct )
{
me = cp ;
ll2 += *lp ;
}
}
if ( num
&& nvc )
{
sum /= nvc ;
sprintf( buf, "%-30.20LG", sum );
for ( cp = buf + strlen( buf ) - 1 ;
cp >= buf && *cp == ' ' ;
cp-- )
{
*cp = NULL ;
}
}
else
{
strcpy( buf, "N/A" );
}
if ( ( ( pfx
&& 0 > printf( "%s|", pfx ) )
|| 0 > printf( "%lu|%lu|%s|%lu|%s|%lu|%s|%s|%s|%lu|%lu\n",
dct, nct,
( dct ? vl[0] : "" ),
( dct ? vct[0] : 0 ),
( dct ? vl[dct - 1] : "" ),
( dct ? vct[dct - 1] : 0 ),
buf, me, mp, mpc, lct ) )
&& ferror( stdout ) )
{
perror( "stdout" );
exit( 1 );
}
exit( 0 );
}
I am not sure whether anybody mentioned to you but when you use a variable under 'awk' that is originally a string (because you piped it as a string from somewhere or you used it as a string in the first place), then you just have to add a number to it and 'awk' converts the result into a number. Usually you add 0 (zero) to it.
Here is an example that should work for you:
wc -w inputfile | cut -c 1-9 | awk '{print $1 + a} a=0'
In my case, the inputfile had 10 words and awk returned 10 at the end. You can easily see this when you add a number to the result (like "1" in the case below)"
wc -w inputfile | cut -c 1-9 | awk 'END {c=$1+a; print c} a=1'
10
11
I think he wanted word frequency, not total word count. He could have used a tool like awk, perl and put 1 or value+1 to a variable for every word in lower case, then dump the variables.
Hey guys,
Does anybody know how to parse a large text file to remove punctuation completely, so it only displays text?
Thanks
look into 'man tr'.