#!/bin/sh
unset LANG LC_CTYPE LC_ALL LC_COLLATE
make
echo

echo "Statistics on input files:"
echo "--------------------------"
echo -n "wolig.dat: "
echo -n `grep " ע" wolig.dat | grep -vc "^#"`
echo -n " noun lines, "
echo -n `grep " ת" wolig.dat | grep -vc "^#"`
echo " adjective lines."
echo -n "woo.dat: "
echo -n `grep " פ" woo.dat | grep -vc "^#"`
echo " verb lines."
echo -n "shemp.dat: "
echo -n `grep " ע" shemp.dat | grep -vc "^#"`
echo " auto-generated gerunds."
echo -n "misc data lines:" `egrep -hcv "^[-#]|^$" extrawords.hif` "extrawords, "
echo -n `grep -hcv "^[-#]" milot.hif` "milot, "
echo -n `grep -hcv "^[-#]" biza-verbs.hif` "bizaverbs, "
echo `grep -hc "^[-#]" biza-nouns.hif` "bizanouns. "

echo

echo "Unique baseword counts:"
echo "-----------------------"
NN=`grep -h " ע" wolig.dat shemp.dat | sed "/^#/d;s/ *#.*$//" | sort -u | wc -l`
NN1=`grep -h " ע" wolig.dat shemp.dat | sed "/^#/d;s/ *#.*$//" | sort -u | grep -vc "ע$"`
NN2=`sed "s/#.*$//" < wolig.dat | egrep ",(זכר|נקבה)" | grep "ע,"| wc -l`
NN3=`grep -h " ע" wolig.dat shemp.dat | sed "/^#/d;s/ *#.*$//" | sort -u | egrep ",(יחיד|רבים|ות|ים|יים|אות)" |wc -l`
echo Nouns: $NN "(of" them, $NN3 need plural hints, $NN1 need inflection hints, $NN2 explicit "gender)."

VV=`grep -c -- ---- verbs.hif`
echo Verbs: $VV

AA=`grep " ת" wolig.dat | grep -v "^#" | sed "s/ *#.*$//" | sort -u | wc -l`
echo Adjectives: $AA

EE=`grep -hv "^[-#]" extrawords.hif milot.hif biza-verbs.hif biza-nouns.hif |  sed "s/ *#.*$//" | tr -d - | sort -u | wc -l`
echo Other words: $EE
echo
echo Total number of base words - `expr $NN + $VV + $AA + $EE`

echo
echo "Final word count:"
echo "-----------------"

# we can count words in hebrew.wgz even without compiling wunzip :)
WW=`zcat hebrew.wgz | tr [0-9] '\012' | grep -vc "^$"`
echo Unique words in hebrew.wgz: $WW
echo "Dictionary file sizes (in bytes):"
wc -c hebrew.wgz*
echo "Memory use (spell-checker only):"
gzip -dc hebrew.wgz | ./find_sizes >/dev/null

# NOTE: to find duplicates in wolig.dat:
# grep " ע" wolig.dat | grep -v "^#"| sed "s/ *#.*$//"|sort |uniq -c | sort -n | less