Logo Search packages:      
Sourcecode: heirloom-mailx version File versions  Download package

junk.c

/*
 * Heirloom mailx - a mail user agent derived from Berkeley Mail.
 *
 * Copyright (c) 2000-2004 Gunnar Ritter, Freiburg i. Br., Germany.
 */
/*
 * Copyright (c) 2004
 *    Gunnar Ritter.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *    This product includes software developed by Gunnar Ritter
 *    and his contributors.
 * 4. Neither the name of Gunnar Ritter nor the names of his contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY GUNNAR RITTER AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL GUNNAR RITTER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#ifndef lint
#ifdef      DOSCCS
static char sccsid[] = "@(#)junk.c  1.73 (gritter) 3/4/06";
#endif
#endif /* not lint */

#include "config.h"
#include "rcv.h"

#include <sys/stat.h>
#include <fcntl.h>
#include <limits.h>
#include <time.h>
#include <unistd.h>
#include <utime.h>

#ifdef      HAVE_MMAP
#include <sys/mman.h>
#else /* !HAVE_MMAP */
#define     mmap(a, b, c, d, e, f)  MAP_FAILED
#define     munmap(a, b)
#endif      /* !HAVE_MMAP */
#ifndef     HAVE_MREMAP
#define     mremap(a, b, c, d)      MAP_FAILED
#endif      /* !HAVE_MREMAP */

#ifndef     MAP_FAILED
#define     MAP_FAILED  ((void *)-1)
#endif      /* !MAP_FAILED */

#include "extern.h"
#include "md5.h"

/*
 * Mail -- a mail program
 *
 * Junk classification, mostly according to Paul Graham's "A Plan for Spam",
 * August 2002, <http://www.paulgraham.com/spam.html>, and his "Better
 * Bayesian Filtering", January 2003, <http://www.paulgraham.com/better.html>.
 *
 * Chained tokens according to Jonathan A. Zdziarski's "Advanced Language
 * Classification using Chained Tokens", February 2004,
 * <http://www.nuclearelephant.com/papers/chained.html>.
 */

#define     DFL   .40
#define     THR   .9
#define     MID   .5

#define     MAX2  0x0000ffff
#define     MAX3  0x00ffffffUL
#define     MAX4  0xffffffffUL

/*
 * The dictionary consists of two files forming a hash table. The hash
 * consists of the first 56 bits of the result of applying MD5 to the
 * input word. This scheme ensures that collisions are unlikely enough
 * to make junk detection work; according to the birthday paradox, a
 * 50 % probability for one single collision is reached at 2^28 entries.
 *
 * To make the chain structure independent from input, the MD5 input is
 * xor'ed with a random number. This makes it impossible that someone uses
 * a carefully crafted message for a denial-of-service attack against the
 * database.
 */
#define     SIZEOF_node 17
#define     OF_node_hash      0     /* first 32 bits of MD5 of word|mangle */
#define     OF_node_next      4     /* bit-negated table index of next node */
#define     OF_node_good      8     /* number of times this appeared in good msgs */
#define     OF_node_bad 11    /* number of times this appeared in bad msgs */
#define     OF_node_prob_O    14    /* table_version<1: precomputed probability */
#define     OF_node_hash2     14    /* upper 3 bytes of MD5 hash */
static char *nodes;

#define     SIZEOF_super      262164
#define     OF_super_size     0     /* allocated nodes in the chain file */
#define     OF_super_used     4     /* used nodes in the chain file */
#define     OF_super_ngood    8     /* number of good messages scanned so far */
#define     OF_super_nbad     12    /* number of bad messages scanned so far */
#define     OF_super_mangle   16    /* used to mangle the MD5 input */
#define     OF_super_bucket   20    /* 65536 bit-negated node indices */
#define     SIZEOF_entry      4
static char *super;

static size_t     super_mmapped;
static size_t     nodes_mmapped;
static int  rw_map;
static int  chained_tokens;

/*
 * Version history
 * ---------------
 * 0  Initial version
 * 1  Fixed the mangling; it was ineffective in version 0.
 *      Hash extended to 56 bits.
 */
static int  table_version;
#define     current_table_version   1

#define     get(e)      \
      ((unsigned)(((char *)(e))[0]&0377) + \
       ((unsigned)(((char *)(e))[1]&0377) << 8) + \
       ((unsigned)(((char *)(e))[2]&0377) << 16))

#define     put(e, n) \
      (((char *)(e))[0] = (n) & 0x0000ff, \
       ((char *)(e))[1] = ((n) & 0x00ff00) >> 8, \
       ((char *)(e))[2] = ((n) & 0xff0000) >> 16)

#define     f2s(d)      (smin(((unsigned)((d) * MAX3)), MAX3))

#define     s2f(s)      ((float)(s) / MAX3)

#define     getn(p)     \
      ((unsigned long)(((char *)(p))[0]&0377) + \
       ((unsigned long)(((char *)(p))[1]&0377) << 8) + \
       ((unsigned long)(((char *)(p))[2]&0377) << 16) + \
       ((unsigned long)(((char *)(p))[3]&0377) << 24))

#define     putn(p, n) \
      (((char *)(p))[0] = (n) & 0x000000ffUL, \
       ((char *)(p))[1] = ((n) & 0x0000ff00UL) >> 8, \
       ((char *)(p))[2] = ((n) & 0x00ff0000UL) >> 16, \
       ((char *)(p))[3] = ((n) & 0xff000000UL) >> 24)

struct lexstat {
      char  *save;
      int   price;
      int   url;
      int   lastc;
      int   hadamp;
      enum loc {
            FROM_LINE   = 0,
            HEADER            = 1,
            BODY        = 2
      } loc;
      enum html {
            HTML_NONE   = 0,
            HTML_TEXT   = 1,
            HTML_TAG    = 2,
            HTML_SKIP   = 3
      } html;
      char  tag[8];
      char  *tagp;
      char  field[LINESIZE];
};

#define     constituent(c, b, i, price, hadamp) \
      ((c) & 0200 || alnumchar(c) || (c) == '\'' || (c) == '"' || \
            (c) == '$' || (c) == '!' || (c) == '_' || \
            (c) == '#' || (c) == '%' || (c) == '&' || \
            ((c) == ';' && hadamp) || \
            ((c) == '-' && !(price)) || \
            (((c) == '.' || (c) == ',' || (c) == '/') && \
             (i) > 0 && digitchar((b)[(i)-1]&0377)))

#define     url_xchar(c) \
      (((c)&0200) == 0 && ((c)&037) != (c) && (c) != 0177 && \
       !spacechar(c) && (c) != '{' && (c) != '}' && (c) != '|' && \
       (c) != '\\' && (c) != '^' && (c) != '~' && (c) != '[' && \
       (c) != ']' && (c) != '`' && (c) != '<' && (c) != '>' && \
       (c) != '#' && (c) != '"')

enum db {
      SUPER = 0,
      NODES = 1
};

enum entry {
      GOOD = 0,
      BAD  = 1
};

static const char README1[] = "\
This is a junk mail database maintained by mailx(1). It does not contain any\n\
of the actual words found in your messages. Instead, parts of MD5 hashes are\n\
used for lookup. It is thus possible to tell if some given word was likely\n\
contained in your mail from examining this data, at best.\n";
static const char README2[] = "\n\
The database files are stored in compress(1) format by default. This saves\n\
some space, but leads to higher processor usage when the database is read\n\
or updated. You can use uncompress(1) on these files if you prefer to store\n\
them in flat form.\n";

static int  verbose;
static int  _debug;
static FILE *sfp, *nfp;
static char *sname, *nname;

static enum okay getdb(int rw);
static void putdb(void);
static void relsedb(void);
static FILE *dbfp(enum db db, int rw, int *compressed, char **fn);
static char *lookup(unsigned long h1, unsigned long h2, int create);
static unsigned long grow(unsigned long size);
static char *nextword(char **buf, size_t *bufsize, size_t *count, FILE *fp,
            struct lexstat *sp, int *stop);
static void join(char **buf, size_t *bufsize, const char *s1, const char *s2);
static void add(const char *word, enum entry entry, struct lexstat *sp,
            int incr);
static enum okay scan(struct message *m, enum entry entry,
            void (*func)(const char *, enum entry, struct lexstat *, int),
            int arg);
static void recompute(void);
static float getprob(char *n);
static int insert(int *msgvec, enum entry entry, int incr);
static void clsf(struct message *m);
static void rate(const char *word, enum entry entry, struct lexstat *sp,
            int unused);
static void dbhash(const char *word, unsigned long *h1, unsigned long *h2);
static void mkmangle(void);

static enum okay 
getdb(int rw)
{
      void  *zp = NULL;
      long  n;
      int   compressed;

      chained_tokens = value("chained-junk-tokens") != NULL;
      if ((sfp = dbfp(SUPER, rw, &compressed, &sname)) == (FILE *)-1)
            return STOP;
      if (sfp && !compressed) {
            super = mmap(NULL, SIZEOF_super,
                        rw!=O_RDONLY ? PROT_READ|PROT_WRITE : PROT_READ,
                        MAP_SHARED, fileno(sfp), 0);
            if (super != MAP_FAILED) {
                  super_mmapped = SIZEOF_super;
                  goto skip;
            }
      }
      super_mmapped = 0;
      super = smalloc(SIZEOF_super);
      if (sfp) {
            if (compressed)
                  zp = zalloc(sfp);
            if ((compressed ? zread(zp, super, SIZEOF_super)
                              != SIZEOF_super :
                        fread(super, 1, SIZEOF_super, sfp)
                              != SIZEOF_super) ||
                        ferror(sfp)) {
                  fprintf(stderr, "Error reading junk mail database.\n");
                  memset(super, 0, SIZEOF_super);
                  mkmangle();
                  if (compressed)
                        zfree(zp);
                  Fclose(sfp);
                  sfp = NULL;
            } else if (compressed)
                  zfree(zp);
      } else {
            memset(super, 0, SIZEOF_super);
            mkmangle();
      }
skip: if ((n = getn(&super[OF_super_size])) == 0) {
            n = 1;
            putn(&super[OF_super_size], 1);
      }
      if (sfp && (nfp = dbfp(NODES, rw, &compressed, &nname)) != NULL) {
            if (nfp == (FILE *)-1) {
                  relsedb();
                  return STOP;
            }
      }
      rw_map = rw;
      if (sfp && nfp && !compressed) {
            nodes = mmap(NULL, n * SIZEOF_node,
                        rw!=O_RDONLY ? PROT_READ|PROT_WRITE : PROT_READ,
                        MAP_SHARED, fileno(nfp), 0);
            if (nodes != MAP_FAILED) {
                  nodes_mmapped = n * SIZEOF_node;
                  return OKAY;
            }
      }
      nodes_mmapped = 0;
      nodes = smalloc(n * SIZEOF_node);
      if (sfp && nfp) {
            if (compressed)
                  zp = zalloc(nfp);
            if ((compressed ? zread(zp, nodes, n * SIZEOF_node)
                        != n * SIZEOF_node :
                        fread(nodes, 1, n * SIZEOF_node, nfp)
                        != n * SIZEOF_node) ||
                        ferror(nfp)) {
                  fprintf(stderr, "Error reading junk mail database.\n");
                  memset(nodes, 0, n * SIZEOF_node);
                  memset(super, 0, SIZEOF_super);
                  mkmangle();
                  putn(&super[OF_super_size], n);
            }
            if (compressed)
                  zfree(zp);
            Fclose(nfp);
            nfp = NULL;
      } else
            memset(nodes, 0, n * SIZEOF_node);
      if (sfp) {
            Fclose(sfp);
            sfp = NULL;
      }
      return OKAY;
}

static void 
putdb(void)
{
      void  *zp;
      int   scomp, ncomp;

      if (!super_mmapped && (sfp = dbfp(SUPER, O_WRONLY, &scomp, &sname))
                  == NULL || sfp == (FILE *)-1)
            return;
      if (!nodes_mmapped && (nfp = dbfp(NODES, O_WRONLY, &ncomp, &nname))
                  == NULL || nfp == (FILE *)-1)
            return;
      if (super_mmapped == 0 || nodes_mmapped == 0)
            holdint();
      /*
       * Use utime() with mmap() since Linux does not update st_mtime
       * reliably otherwise.
       */
      if (super_mmapped)
            utime(sname, NULL);
      else if (scomp) {
            zp = zalloc(sfp);
            zwrite(zp, super, SIZEOF_super);
            zfree(zp);
            trunc(sfp);
      } else
            fwrite(super, 1, SIZEOF_super, sfp);
      if (nodes_mmapped)
            utime(nname, NULL);
      else if (ncomp) {
            zp = zalloc(nfp);
            zwrite(zp, nodes, getn(&super[OF_super_size]) * SIZEOF_node);
            zfree(zp);
            trunc(nfp);
      } else
            fwrite(nodes, 1,
                  getn(&super[OF_super_size]) * SIZEOF_node, nfp);
      if (super_mmapped == 0 || nodes_mmapped == 0)
            relseint();
}

static void
relsedb(void)
{
      if (super_mmapped) {
            munmap(super, super_mmapped);
            super_mmapped = 0;
      } else
            free(super);
      if (nodes_mmapped) {
            munmap(nodes, nodes_mmapped);
            nodes_mmapped = 0;
      } else
            free(nodes);
      if (sfp && sfp != (FILE *)-1) {
            Fclose(sfp);
            sfp = NULL;
      }
      if (nfp && nfp != (FILE *)-1) {
            Fclose(nfp);
            nfp = NULL;
      }
}

static FILE *
dbfp(enum db db, int rw, int *compressed, char **fn)
{
      FILE  *fp, *rp;
      char  *dir;
      struct flock      flp;
      char *sfx[][2] = {
            { "super",  "nodes" },
            { "super1", "nodes1" }
      };
      char **sf;
      char *zfx[][2] = {
            { "super.Z",      "nodes.Z" },
            { "super1.Z",     "nodes1.Z" }
      };
      char **zf;
      int   n;

      if ((dir = value("junkdb")) == NULL) {
            fprintf(stderr, "No junk mail database specified. "
                        "Set the junkdb variable.\n");
            return (FILE *)-1;
      }
      dir = expand(dir);
      if (makedir(dir) == STOP) {
            fprintf(stderr, "Cannot create directory \"%s\"\n.", dir);
            return (FILE *)-1;
      }
      if (rw!=O_WRONLY)
            table_version = current_table_version;
loop: sf = sfx[table_version];
      zf = zfx[table_version];
      *fn = salloc((n = strlen(dir)) + 40);
      strcpy(*fn, dir);
      (*fn)[n] = '/';
      *compressed = 0;
      strcpy(&(*fn)[n+1], sf[db]);
      if ((fp = Fopen(*fn, rw!=O_RDONLY ? "r+" : "r")) != NULL)
            goto okay;
      *compressed = 1;
      strcpy(&(*fn)[n+1], zf[db]);
      if ((fp = Fopen(*fn, rw ? "r+" : "r")) == NULL &&
                  rw==O_WRONLY ? (fp = Fopen(*fn, "w+")) == NULL : 0) {
            fprintf(stderr, "Cannot open junk mail database \"%s\".\n",*fn);
            return NULL;
      }
      if (rw==O_WRONLY) {
            strcpy(&(*fn)[n+1], "README");
            if (access(*fn, F_OK) < 0 && (rp = Fopen(*fn, "w")) != NULL) {
                  fputs(README1, rp);
                  fputs(README2, rp);
                  Fclose(rp);
            }
      } else if (fp == NULL) {
            if (table_version > 0) {
                  table_version--;
                  goto loop;
            } else
                  table_version = current_table_version;
      }
okay: if (fp) {
            flp.l_type = rw!=O_RDONLY ? F_WRLCK : F_RDLCK;
            flp.l_start = 0;
            flp.l_len = 0;
            flp.l_whence = SEEK_SET;
            fcntl(fileno(fp), F_SETLKW, &flp);
      }
      return fp;
}

static char *
lookup(unsigned long h1, unsigned long h2, int create)
{
      char  *n, *lastn = NULL;
      unsigned long     c, lastc = MAX4, used, size;

      used = getn(&super[OF_super_used]);
      size = getn(&super[OF_super_size]);
      c = ~getn(&super[OF_super_bucket + (h1&MAX2)*SIZEOF_entry]);
      n = &nodes[c*SIZEOF_node];
      while (c < used) {
            if (getn(&n[OF_node_hash]) == h1 &&
                        (table_version < 1 ? 1 :
                        get(&n[OF_node_hash2]) == h2))
                  return n;
            lastc = c;
            lastn = n;
            c = ~getn(&n[OF_node_next]);
            n = &nodes[c*SIZEOF_node];
      }
      if (create) {
            if (used >= size) {
                  if ((size = grow(size)) == 0)
                        return NULL;
                  lastn = &nodes[lastc*SIZEOF_node];
            }
            putn(&super[OF_super_used], used+1);
            n = &nodes[used*SIZEOF_node];
            putn(&n[OF_node_hash], h1);
            put(&n[OF_node_hash2], h2);
            if (lastc < used)
                  putn(&lastn[OF_node_next], ~used);
            else
                  putn(&super[OF_super_bucket + (h1&MAX2)*SIZEOF_entry],
                              ~used);
            return n;
      } else
            return NULL;
}

static unsigned long
grow(unsigned long size)
{
      unsigned long     incr, newsize;
      void  *onodes;

      incr = size > MAX2 ? MAX2 : size;
      newsize = size + incr;
      if (newsize > MAX4-MAX2) {
      oflo: fprintf(stderr, "Junk mail database overflow.\n");
            return 0;
      }
      if (nodes_mmapped) {
            if (lseek(fileno(nfp), newsize*SIZEOF_node-1, SEEK_SET)
                        == (off_t)-1 || write(fileno(nfp),"\0",1) != 1)
                  goto oflo;
            onodes = nodes;
            if ((nodes = mremap(nodes, nodes_mmapped, newsize*SIZEOF_node,
                        MREMAP_MAYMOVE)) == MAP_FAILED) {
                  if ((nodes = mmap(NULL, newsize*SIZEOF_node,
                                    rw_map!=O_RDONLY ?
                                          PROT_READ|PROT_WRITE :
                                          PROT_READ,
                                    MAP_SHARED, fileno(nfp), 0))
                              == MAP_FAILED) {
                        nodes = onodes;
                        goto oflo;
                  }
                  munmap(onodes, nodes_mmapped);
            }
            nodes_mmapped = newsize*SIZEOF_node;
      } else {
            nodes = srealloc(nodes, newsize*SIZEOF_node);
            memset(&nodes[size*SIZEOF_node], 0, incr*SIZEOF_node);
      }
      size = newsize;
      putn(&super[OF_super_size], size);
      return size;
}

#define     SAVE(c)     { \
      if (i+j >= (long)*bufsize-4) \
            *buf = srealloc(*buf, *bufsize += 32); \
      (*buf)[j+i] = (c); \
      i += (*buf)[j+i] != '\0'; \
}

static char *
nextword(char **buf, size_t *bufsize, size_t *count, FILE *fp,
            struct lexstat *sp, int *stop)
{
      int   c, i, j, k;
      char  *cp, *cq;

loop: *stop = 0;
      sp->hadamp = 0;
      if (sp->save) {
            i = j = 0;
            for (cp = sp->save; *cp; cp++) {
                  SAVE(*cp&0377)
            }
            SAVE('\0')
            free(sp->save);
            sp->save = NULL;
            goto out;
      }
      if (sp->loc == FROM_LINE)
            while (*count > 0 && (c = getc(fp)) != EOF) {
                  sp->lastc = c;
                  if (c == '\n') {
                        sp->loc = HEADER;
                        break;
                  }
            }
      i = 0;
      j = 0;
      if (sp->loc == HEADER && sp->field[0]) {
      field:      cp = sp->field;
            do {
                  c = *cp&0377;
                  SAVE(c)
                  cp++;
            } while (*cp);
            j = i;
            i = 0;
      }
      if (sp->price) {
            sp->price = 0;
            SAVE('$')
      }
      while (*count > 0 && (c = getc(fp)) != EOF) {
            (*count)--;
            if (c == '\0' && table_version >= 1) {
                  sp->loc = HEADER;
                  sp->lastc = '\n';
                  *stop = 1;
                  continue;
            }
            if (c == '\b' && table_version >= 1) {
                  sp->html = HTML_TEXT;
                  continue;
            }
            if (c == '<' && sp->html == HTML_TEXT) {
                  sp->html = HTML_TAG;
                  sp->tagp = sp->tag;
                  continue;
            }
            if (sp->html == HTML_TAG) {
                  if (spacechar(c)) {
                        *sp->tagp = '\0';
                        if (!asccasecmp(sp->tag, "a") ||
                                    !asccasecmp(sp->tag, "img") ||
                                    !asccasecmp(sp->tag, "font") ||
                                    !asccasecmp(sp->tag, "span") ||
                                    !asccasecmp(sp->tag, "meta") ||
                                    !asccasecmp(sp->tag, "table") ||
                                    !asccasecmp(sp->tag, "tr") ||
                                    !asccasecmp(sp->tag, "td") ||
                                    !asccasecmp(sp->tag, "p"))
                              sp->html = HTML_TEXT;
                        else
                              sp->html = HTML_SKIP;
                  } else if (c == '>') {
                        sp->html = HTML_TEXT;
                        continue;
                  } else {
                        if (sp->tagp - sp->tag < sizeof sp->tag - 1)
                              *sp->tagp++ = c;
                        continue;
                  }
            }
            if (sp->html == HTML_SKIP) {
                  if (c == '>')
                        sp->html = HTML_TEXT;
                  continue;
            }
            if (c == '$' && i == 0)
                  sp->price = 1;
            if (sp->loc == HEADER && sp->lastc == '\n') {
                  if (!spacechar(c)) {
                        k = 0;
                        while (k < sizeof sp->field - 3) {
                              sp->field[k++] = c;
                              if (*count <= 0 ||
                                          (c = getc(fp)) == EOF)
                                    break;
                              if (spacechar(c) || c == ':') {
                                    ungetc(c, fp);
                                    break;
                              }
                              sp->lastc = c;
                              (*count)--;
                        }
                        sp->field[k++] = '*';
                        sp->field[k] = '\0';
                        j = 0;
                        *stop = 1;
                        goto field;
                  } else if (c == '\n') {
                        j = 0;
                        sp->loc = BODY;
                        sp->html = HTML_NONE;
                        *stop = 1;
                  }
            }
            if (sp->url) {
                  if (!url_xchar(c)) {
                        sp->url = 0;
                        cp = sp->save = smalloc(i+6);
                        for (cq = "HOST*"; *cq; cq++)
                              *cp++ = *cq;
                        for (cq = &(*buf)[j]; *cq != ':'; cq++);
                        cq += 3;    /* skip "://" */
                        while (cq < &(*buf)[i+j] &&
                                    (alnumchar(*cq&0377) ||
                                     *cq == '.' || *cq == '-'))
                              *cp++ = *cq++;
                        *cp = '\0';
                        *stop = 1;
                        break;
                  }
                  SAVE(c)
            } else if (constituent(c, *buf, i+j, sp->price, sp->hadamp) ||
                        sp->loc == HEADER && c == '.' &&
                        asccasecmp(sp->field, "subject*")) {
                  if (c == '&')
                        sp->hadamp = 1;
                  SAVE(c)
            } else if (i > 0 && c == ':' && *count > 2) {
                  if ((c = getc(fp)) != '/') {
                        ungetc(c, fp);
                        break;
                  }
                  (*count)--;
                  if ((c = getc(fp)) != '/') {
                        ungetc(c, fp);
                        break;
                  }
                  (*count)--;
                  sp->url = 1;
                  SAVE('\0')
                  cp = savestr(*buf);
                  j = i = 0;
                  for (cq = "URL*"; *cq; cq++) {
                        SAVE(*cq&0377)
                  }
                  j = i;
                  i = 0;
                  do {
                        if (alnumchar(*cp&0377)) {
                              SAVE(*cp&0377)
                        } else
                              i = 0;
                  } while (*++cp);
                  for (cq = "://"; *cq; cq++) {
                        SAVE(*cq&0377)
                  }
            } else if (i > 1 && ((*buf)[i+j-1] == ',' ||
                         (*buf)[i+j-1] == '.') && !digitchar(c)) {
                  i--;
                  ungetc(c, fp);
                  (*count)++;
                  break;
            } else if (i > 0) {
                  sp->lastc = c;
                  break;
            }
            sp->lastc = c;
      }
out:  if (i > 0) {
            SAVE('\0')
            c = 0;
            for (k = 0; k < i; k++)
                  if (digitchar((*buf)[k+j]&0377))
                        c++;
                  else if (!alphachar((*buf)[k+j]&0377) &&
                              (*buf)[k+j] != '$') {
                        c = 0;
                        break;
                  }
            if (c == i)
                  goto loop;
            /*
             * Including the results of other filtering software (the
             * 'X-Spam' fields) might seem tempting, but will also rate
             * their false negatives good with this filter. Therefore
             * these fields are ignored.
             *
             * Handling 'Received' fields is difficult since they include
             * lots of both useless and interesting words for our purposes.
             */
            if (sp->loc == HEADER &&
                        (asccasecmp(sp->field, "message-id*") == 0 ||
                         asccasecmp(sp->field, "references*") == 0 ||
                         asccasecmp(sp->field, "in-reply-to*") == 0 ||
                         asccasecmp(sp->field, "status*") == 0 ||
                         asccasecmp(sp->field, "x-status*") == 0 ||
                         asccasecmp(sp->field, "date*") == 0 ||
                         asccasecmp(sp->field, "delivery-date*") == 0 ||
                         ascncasecmp(sp->field, "x-spam", 6) == 0 ||
                         ascncasecmp(sp->field, "x-pstn", 6) == 0 ||
                         ascncasecmp(sp->field, "x-scanned", 9) == 0 ||
                         asccasecmp(sp->field, "received*") == 0 &&
                              ((2*c > i) || i < 4 ||
                              asccasestr(*buf, "localhost") != NULL)))
                  goto loop;
            return *buf;
      }
      return NULL;
}

#define     JOINCHECK   if (i >= *bufsize) \
                        *buf = srealloc(*buf, *bufsize += 32)
static void
join(char **buf, size_t *bufsize, const char *s1, const char *s2)
{
      int   i = 0;

      while (*s1) {
            JOINCHECK;
            (*buf)[i++] = *s1++;
      }
      JOINCHECK;
      (*buf)[i++] = ' ';
      do {
            JOINCHECK;
            (*buf)[i++] = *s2;
      } while (*s2++);
}

/*ARGSUSED3*/
static void
add(const char *word, enum entry entry, struct lexstat *sp, int incr)
{
      unsigned    c;
      unsigned long     h1, h2;
      char  *n;

      dbhash(word, &h1, &h2);
      if ((n = lookup(h1, h2, 1)) != NULL) {
            switch (entry) {
            case GOOD:
                  c = get(&n[OF_node_good]);
                  if (incr>0 && c<MAX3-incr || incr<0 && c>=-incr) {
                        c += incr;
                        put(&n[OF_node_good], c);
                  }
                  break;
            case BAD:
                  c = get(&n[OF_node_bad]);
                  if (incr>0 && c<MAX3-incr || incr<0 && c>=-incr) {
                        c += incr;
                        put(&n[OF_node_bad], c);
                  }
                  break;
            }
      }
}

static enum okay 
scan(struct message *m, enum entry entry,
            void (*func)(const char *, enum entry, struct lexstat *, int),
            int arg)
{
      FILE  *fp;
      char  *buf0 = NULL, *buf1 = NULL, *buf2 = NULL, **bp, *cp;
      size_t      bufsize0 = 0, bufsize1 = 0, bufsize2 = 0, *zp, count;
      struct lexstat    *sp;
      int   stop;

      if ((fp = Ftemp(&cp, "Ra", "w+", 0600, 1)) == NULL) {
            perror("tempfile");
            return STOP;
      }
      rm(cp);
      Ftfree(&cp);
      if (send(m, fp, NULL, NULL, SEND_TOFLTR, NULL) < 0) {
            Fclose(fp);
            return STOP;
      }
      fflush(fp);
      rewind(fp);
      sp = scalloc(1, sizeof *sp);
      count = fsize(fp);
      bp = &buf0;
      zp = &bufsize0;
      while (nextword(bp, zp, &count, fp, sp, &stop) != NULL) {
            (*func)(*bp, entry, sp, arg);
            if (chained_tokens && buf0 && *buf0 && buf1 && *buf1 && !stop) {
                  join(&buf2, &bufsize2, bp == &buf1 ? buf0 : buf1, *bp);
                  (*func)(buf2, entry, sp, arg);
            }
            bp = bp == &buf1 ? &buf0 : &buf1;
            zp = zp == &bufsize1 ? &bufsize0 : &bufsize1;
      }
      free(buf0);
      free(buf1);
      free(buf2);
      free(sp);
      Fclose(fp);
      return OKAY;
}

static void
recompute(void)
{
      unsigned long     used, i;
      unsigned    s;
      char  *n;
      float p;

      used = getn(&super[OF_super_used]);
      for (i = 0; i < used; i++) {
            n = &nodes[i*SIZEOF_node];
            p = getprob(n);
            s = f2s(p);
            put(&n[OF_node_prob_O], s);
      }
}

static float
getprob(char *n)
{
      unsigned long     ngood, nbad;
      unsigned    g, b;
      float p, BOT, TOP;

      ngood = getn(&super[OF_super_ngood]);
      nbad = getn(&super[OF_super_nbad]);
      if (ngood + nbad >= 18000) {
            BOT = .0001;
            TOP = .9999;
      } else if (ngood + nbad >= 9000) {
            BOT = .001;
            TOP = .999;
      } else {
            BOT = .01;
            TOP = .99;
      }
      g = get(&n[OF_node_good]) * 2;
      b = get(&n[OF_node_bad]);
      if (g + b >= 5) {
            p = smin(1.0, nbad ? (float)b/nbad : 0.0) /
                  (smin(1.0, ngood ? (float)g/ngood : 0.0) +
                   smin(1.0, nbad ? (float)b/nbad : 0.0));
            p = smin(TOP, p);
            p = smax(BOT, p);
      } else if (g == 0 && b == 0)
            p = DFL;
      else
            p = 0;
      return p;
}

static int 
insert(int *msgvec, enum entry entry, int incr)
{
      int   *ip;
      unsigned long     u = 0;

      verbose = value("verbose") != NULL;
      if (getdb(O_RDWR) != OKAY)
            return 1;
      switch (entry) {
      case GOOD:
            u = getn(&super[OF_super_ngood]);
            break;
      case BAD:
            u = getn(&super[OF_super_nbad]);
            break;
      }
      for (ip = msgvec; *ip; ip++) {
            setdot(&message[*ip-1]);
            if (incr > 0 && u == MAX4-incr+1) {
                  fprintf(stderr, "Junk mail database overflow.\n");
                  break;
            } else if (incr < 0 && -incr > u) {
                  fprintf(stderr, "Junk mail database underflow.\n");
                  break;
            }
            u += incr;
            if (entry == GOOD && incr > 0 || entry == BAD && incr < 0)
                  message[*ip-1].m_flag &= ~MJUNK;
            else
                  message[*ip-1].m_flag |= MJUNK;
            scan(&message[*ip-1], entry, add, incr);
      }
      switch (entry) {
      case GOOD:
            putn(&super[OF_super_ngood], u);
            break;
      case BAD:
            putn(&super[OF_super_nbad], u);
            break;
      }
      if (table_version < 1)
            recompute();
      putdb();
      relsedb();
      return 0;
}

int
cgood(void *v)
{
      return insert(v, GOOD, 1);
}

int
cjunk(void *v)
{
      return insert(v, BAD, 1);
}

int
cungood(void *v)
{
      return insert(v, GOOD, -1);
}

int
cunjunk(void *v)
{
      return insert(v, BAD, -1);
}

int 
cclassify(void *v)
{
      int   *msgvec = v, *ip;

      verbose = value("verbose") != NULL;
      _debug = debug || value("debug") != NULL;
      if (getdb(O_RDONLY) != OKAY)
            return 1;
      for (ip = msgvec; *ip; ip++) {
            setdot(&message[*ip-1]);
            clsf(&message[*ip-1]);
      }
      relsedb();
      return 0;
}

#define     BEST  15
static struct {
      float dist;
      float prob;
      char  *word;
      unsigned long     hash1;
      unsigned long     hash2;
      enum loc    loc;
} best[BEST];

static void 
clsf(struct message *m)
{
      int   i;
      float a = 1, b = 1, r;

      if (verbose)
            fprintf(stderr, "Examining message %d\n", m - &message[0] + 1);
      for (i = 0; i < BEST; i++) {
            best[i].dist = 0;
            best[i].prob = -1;
      }
      if (scan(m, -1, rate, 0) != OKAY)
            return;
      if (best[0].prob == -1) {
            if (verbose)
                  fprintf(stderr, "No information found.\n");
            m->m_flag &= ~MJUNK;
            return;
      }
      for (i = 0; i < BEST; i++) {
            if (best[i].prob == -1)
                  break;
            if (verbose)
                  fprintf(stderr, "Probe %2d: \"%s\", hash=%lu:%lu "
                        "prob=%.4g dist=%.4g\n",
                        i+1, prstr(best[i].word),
                        best[i].hash1, best[i].hash2,
                        best[i].prob, best[i].dist);
            a *= best[i].prob;
            b *= 1 - best[i].prob;
      }
      r = a+b > 0 ? a / (a+b) : 0;
      if (verbose)
            fprintf(stderr, "Junk probability of message %d: %g\n",
                        m - &message[0] + 1, r);
      if (r > THR)
            m->m_flag |= MJUNK;
      else
            m->m_flag &= ~MJUNK;
}

/*ARGSUSED4*/
static void
rate(const char *word, enum entry entry, struct lexstat *sp, int unused)
{
      char  *n;
      unsigned long     h1, h2;
      float p, d;
      int   i, j;

      dbhash(word, &h1, &h2);
      if ((n = lookup(h1, h2, 0)) != NULL) {
            p = getprob(n);
      } else
            p = DFL;
      if (_debug)
            fprintf(stderr, "h=%lu:%lu g=%u b=%u p=%.4g %s\n", h1, h2,
                        n ? get(&n[OF_node_good]) : 0,
                        n ? get(&n[OF_node_bad]) : 0,
                        p, prstr(word));
      if (p == 0)
            return;
      d = p >= MID ? p - MID : MID - p;
      if (d >= best[BEST-1].dist)
            for (i = 0; i < BEST; i++) {
                  if (h1 == best[i].hash1 && h2 == best[i].hash2)
                        break;
                  /*
                   * This selection prefers words from the end of the
                   * header and from the start of the body. It does
                   * probably not matter much at all, but gives at
                   * least the most interesting verbose output.
                   */
                  if (d > best[i].dist || best[i].loc == HEADER &&
                              d >= best[i].dist) {
                        for (j = BEST-2; j >= i; j--)
                              best[j+1] = best[j];
                        best[i].dist = d;
                        best[i].prob = p;
                        best[i].word = savestr(word);
                        best[i].hash1 = h1;
                        best[i].hash2 = h2;
                        best[i].loc = sp->loc;
                        break;
                  }
            }
}

static void
dbhash(const char *word, unsigned long *h1, unsigned long *h2)
{
      unsigned char     digest[16];
      MD5_CTX     ctx;

      MD5Init(&ctx);
      MD5Update(&ctx, (unsigned char *)word, strlen(word));
      if (table_version >= 1)
            MD5Update(&ctx, (unsigned char *)&super[OF_super_mangle], 4);
      MD5Final(digest, &ctx);
      *h1 = getn(digest);
      if (table_version < 1) {
            *h1 ^= getn(&super[OF_super_mangle]);
            *h2 = 0;
      } else
            *h2 = get(&digest[4]);
}

/*
 * The selection of the value for mangling is not critical. It is practically
 * impossible for any person to determine the exact time when the database
 * was created first (without looking at the database, which would reveal the
 * value anyway), so we just use this. The MD5 hash here ensures that each
 * single second gives a completely different mangling value (which is not
 * necessary anymore if table_version>=1, but does not hurt).
 */
static void 
mkmangle(void)
{
      union {
            time_t      t;
            char  c[16];
      } u;
      unsigned long     s;
      unsigned char     digest[16];
      MD5_CTX     ctx;

      memset(&u, 0, sizeof u);
      time(&u.t);
      MD5Init(&ctx);
      MD5Update(&ctx, (unsigned char *)u.c, sizeof u.c);
      MD5Final(digest, &ctx);
      s = getn(digest);
      putn(&super[OF_super_mangle], s);
}

int
cprobability(void *v)
{
      char  **args = v;
      unsigned long     used, ngood, nbad;
      unsigned long     h1, h2;
      unsigned    g, b;
      float p, d;
      char  *n;

      if (*args == NULL) {
            fprintf(stderr, "No words given.\n");
            return 1;
      }
      if (getdb(O_RDONLY) != OKAY)
            return 1;
      used = getn(&super[OF_super_used]);
      ngood = getn(&super[OF_super_ngood]);
      nbad = getn(&super[OF_super_nbad]);
      printf("Database statistics: tokens=%lu ngood=%lu nbad=%lu\n",
                  used, ngood, nbad);
      do {
            dbhash(*args, &h1, &h2);
            printf("\"%s\", hash=%lu:%lu ", *args, h1, h2);
            if ((n = lookup(h1, h2, 0)) != NULL) {
                  g = get(&n[OF_node_good]);
                  b = get(&n[OF_node_bad]);
                  printf("good=%u bad=%u ", g, b);
                  p = getprob(n);
                  if (p != 0) {
                        d = p >= MID ? p - MID : MID - p;
                        printf("prob=%.4g dist=%.4g", p, d);
                  } else
                        printf("too infrequent");
            } else
                  printf("not in database");
            putchar('\n');
      } while (*++args);
      relsedb();
      return 0;
}

Generated by  Doxygen 1.6.0   Back to index