/*
 * lm.h - a small word-level n-gram language model with back-off (C89).
 *
 * The model is a Markov chain over word tokens: trigram counts with back-off
 * to bigram and unigram. It is deliberately simple and honest about it - it is
 * not a neural network; it predicts the next word from the previous one or two.
 *
 * DeepSeek on Obsidian. Author: DeepSeek (deepseek-v4.1-flash), Oct 2026.
 */
#ifndef LM_H
#define LM_H

#include <stdio.h>

typedef struct LM LM;

typedef struct {
    int vocab;                       /* distinct tokens */
    unsigned long long tokens;       /* tokens seen in training */
    long distinct_bigrams;
    long distinct_trigrams;
    unsigned long long corpus_bytes;
} LMStats;

LM  *lm_create(void);
void lm_free(LM *m);

/* Tokenise one document (context does not cross calls) and count n-grams. */
void lm_add_text(LM *m, const char *text, long len);

/* Build the unigram distribution. Call once after all training. */
void lm_finish(LM *m);

void lm_stats(const LM *m, LMStats *s);
int  lm_vocab(const LM *m);
const char *lm_word(const LM *m, int id);

/* Generate up to max_words tokens into out (NUL terminated). Returns the
 * number of tokens produced. */
int  lm_generate(LM *m, const char *seed, int max_words, char *out, int outsz);

#endif /* LM_H */