/*
 * lmtrain.c - train the n-gram language model and report statistics.
 *
 *   lmtrain corpus.txt [more.txt ...]
 *
 * Documents are separated by blank lines, so n-gram contexts do not cross
 * document boundaries.
 *
 * DeepSeek on Obsidian, C89.
 * Author: DeepSeek (deepseek-v4.1-flash), October 2026.
 */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "lm.h"

static void load_file(LM *m, const char *path)
{
    FILE *f = fopen(path, "r");
    char line[4096];
    char buf[65536];
    long bl = 0;

    if (f == (FILE *)0) {
        fprintf(stderr, "lmtrain: cannot open %s\n", path);
        return;
    }
    while (fgets(line, (int)sizeof line, f)) {
        const char *p = line;
        int blank = 1;
        long ll;
        while (*p) {
            if (*p != ' ' && *p != '\t' && *p != '\n' && *p != '\r') { blank = 0; break; }
            p++;
        }
        if (blank) {
            if (bl > 0) { lm_add_text(m, buf, bl); bl = 0; }
            continue;
        }
        ll = (long)strlen(line);
        if (bl + ll < (long)sizeof buf) {
            memcpy(buf + bl, line, (size_t)ll);
            bl += ll;
        } else {
            lm_add_text(m, buf, bl);
            bl = 0;
            if (ll < (long)sizeof buf) { memcpy(buf, line, (size_t)ll); bl = ll; }
        }
    }
    if (bl > 0) lm_add_text(m, buf, bl);
    fclose(f);
}

int main(int argc, char **argv)
{
    LM *m;
    LMStats s;
    clock_t t0, t1;
    double secs;
    char out[8192];
    int i;

    if (argc < 2) {
        fprintf(stderr, "usage: %s corpus.txt [more.txt ...]\n", argv[0]);
        return 2;
    }

    m = lm_create();
    t0 = clock();
    for (i = 1; i < argc; i++)
        load_file(m, argv[i]);
    lm_finish(m);
    t1 = clock();
    secs = (double)(t1 - t0) / (double)CLOCKS_PER_SEC;

    lm_stats(m, &s);
    printf("corpus_bytes   %lu\n", (unsigned long)s.corpus_bytes);
    printf("tokens         %lu\n", (unsigned long)s.tokens);
    printf("vocab          %d\n", s.vocab);
    printf("distinct bigrams  %ld\n", s.distinct_bigrams);
    printf("distinct trigrams %ld\n", s.distinct_trigrams);
    printf("train_seconds  %.2f\n", secs);
    printf("train_tokens_per_sec %.0f\n",
           secs > 0.0 ? (double)s.tokens / secs : 0.0);

    /* generation throughput */
    {
        clock_t g0, g1;
        long words = 0;
        int k;
        g0 = clock();
        for (k = 0; k < 2000; k++) {
            words += lm_generate(m, "the system", 50, out, (int)sizeof out);
        }
        g1 = clock();
        {
            double gs = (double)(g1 - g0) / (double)CLOCKS_PER_SEC;
            printf("gen_seconds    %.2f\n", gs);
            printf("gen_words_per_sec %.0f\n", gs > 0.0 ? (double)words / gs : 0.0);
        }
    }

    printf("\n--- samples ---\n");
    for (i = 0; i < 3; i++) {
        lm_generate(m, "", 60, out, (int)sizeof out);
        printf("* %s\n\n", out);
    }
    lm_generate(m, "the command", 60, out, (int)sizeof out);
    printf("* the command %s\n", out);

    lm_free(m);
    return 0;
}