/* checkwordfreq.c   by Michael Thorpe   2006-03-18 */

#include <ctype.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <gdbm.h>
#include <errno.h>
#include <unistd.h>

#define MAX_TOKEN 60
#define NUM_WORDS 15
#define MIN_WORDLEN 3
#define EPSILON 0.00000001
/* If >0, checkwordfreq will only look at the first HOW_MUCH_TO_CHECK bytes */
#define HOW_MUCH_TO_CHECK 10000

static char token[MAX_TOKEN+1];
static unsigned long bademails,goodemails;
static double probs[NUM_WORDS];
static double deviations[NUM_WORDS];
static char *(words[NUM_WORDS]);
static int verbose=0;

static unsigned long get(GDBM_FILE dbf,char *word) {
	datum key,value;
	int i;
	unsigned long val=0;

	key.dptr=word;
	key.dsize=strlen(word);
	value=gdbm_fetch(dbf,key);
	if(!value.dptr)
		return(0);
	for(i=0;i<value.dsize && i<sizeof(val);i++)
		val |= ((unsigned char *)value.dptr)[i] << (8*i);
	free(value.dptr);
	return(val);
}

static int doit(FILE *f,GDBM_FILE bad,GDBM_FILE good) {
	unsigned long g,b;
	int c,i;
	double pb,pg,prob;
#if HOW_MUCH_TO_CHECK
	unsigned int l=0;
#endif

	for(i=0;i<NUM_WORDS;i++) {
		probs[i]=0;
		deviations[i]=0;
		words[i]=0;
	}
	i=0;
	while(!feof(f)) {
		c=fgetc(f);
		if(c==EOF && !i)
			break;
#if HOW_MUCH_TO_CHECK
		if(HOW_MUCH_TO_CHECK<=++l)
			break;
#endif
		if(isalnum(c) || c=='\'' || c=='-' || c=='$') {
			token[i]=tolower(c);
			if(i<MAX_TOKEN) {
				i++;
				continue;
			}
		}
		if(i==MAX_TOKEN || i<MIN_WORDLEN)
			i=0;
		if(!i)
			continue;
		token[i]='\0';
		g=2*get(good,token);
		b=get(bad,token);
		if(g+b<5) {
			prob=.2;
		} else {
			pb=(double)b/(double)bademails;
			if(pb>1)
				pb=1;
			pg=(double)g/(double)goodemails;
			if(pg>1)
				pg=1;
			prob=pb/(pb+pg);
			if(prob>.99)
				prob=.99;
			if(prob<.01)
				prob=.01;
		}
		pg=prob-.5;
		if(pg<0)
			pg=-pg;
		if(deviations[NUM_WORDS-1]<pg+EPSILON) {
			for(i=NUM_WORDS-1;i>0;i--) {
				if(deviations[i-1]>pg+EPSILON)
					break;
				if(words[i-1] && fabs(deviations[i-1]-pg)<EPSILON && 0<strcmp(token,words[i-1]))
					break;
			}
			if(!words[i] || strcmp(token,words[i])) {
				memmove(deviations+i+1,deviations+i,(NUM_WORDS-1-i)*sizeof(double));
				deviations[i]=pg;
				memmove(probs+i+1,probs+i,(NUM_WORDS-1-i)*sizeof(double));
				probs[i]=prob;
				free(words[NUM_WORDS-1]);
				memmove(words+i+1,words+i,(NUM_WORDS-1-i)*sizeof(char *));
				words[i]=strdup(token);
				if(!words[i]) {
					perror("strdup");
					return(1);
				}
			}
		}
		i=0;
	}
	pg=pb=1;
	for(i=0;i<NUM_WORDS;i++) {
		pb*=probs[i];
		pg*=(1-probs[i]);
		if(verbose)
			printf("%f\t%s\n",probs[i],words[i]);
	}
	printf("%f\n",pb/(pb+pg));
	return(0);
}

int main(int argc,char **argv) {
	GDBM_FILE bad,good;
	int goterror=0;

	if(argc>1 && argv[1][0]=='-' && argv[1][1]=='v' && !argv[1][2])
		verbose=1;
	if(argc != 3+verbose) {
		fprintf(stderr,"usage: checkwordfreq [-v] <bad_db> <good_db>\n");
		return(1);
	}
	do {
		bad=gdbm_open(argv[1+verbose],0,GDBM_READER,0,0);
	} while(!bad && EAGAIN==errno && (sleep(1) || 1));
	if(!bad) {
		perror("gdbm_open");
		return(1);
	}
	do {
		good=gdbm_open(argv[2+verbose],0,GDBM_READER,0,0);
	} while(!good && EAGAIN==errno && (sleep(1) || 1));
	if(!good) {
		perror("gdbm_open");
		return(1);
	}
	bademails=get(bad," ");
	goodemails=get(good," ");
	if(!bademails || !goodemails) {
		fprintf(stderr,"Must have at least 1 each good and bad email!\n");
		return(1);
	}
	if(doit(stdin,bad,good)) {
		perror("stdin");
		goterror=1;
	}
	gdbm_close(good);
	gdbm_close(bad);
	return(goterror);
}