#!/usr/bin/perl
# mkcorpus.pl - turn nroff man-page sources into a plain-text corpus.
#
#   perl mkcorpus.pl /usr/share/man/man*/* > corpus.txt
#
# Strips troff requests and escapes, keeps the readable text, and separates
# documents with blank lines (the trainer resets n-gram context there).
#
# DeepSeek (deepseek-v4.1-flash), October 2026.

my $prev_blank = 0;

while (my $line = <>) {
    chomp $line;

    # formatted pages (cat*) overstrike with backspaces: "x\bx" -> "x"
    $line =~ s/.\x08//g;
    $line =~ s/[\x00-\x08\x0b\x0c\x0e-\x1f]//g;

    # comment lines
    next if $line =~ /^\.\\"/;

    # drop a leading request name, keep any argument text (".B ps" -> "ps")
    $line =~ s/^\.[A-Za-z][A-Za-z0-9]*[ \t]*//;

    # fonts, string macros, special characters, size/space escapes
    $line =~ s/\\f[BIRP]//g;
    $line =~ s/\\f\([A-Za-z0-9]{2}//g;
    $line =~ s/\\\*\(..//g;
    $line =~ s/\\\*[A-Za-z]{2}//g;
    $line =~ s/\\n\(..//g;
    $line =~ s/\\n.//g;
    $line =~ s/\\v'[^']*'//g;
    $line =~ s/\\h'[^']*'//g;
    $line =~ s/\\l'[^']*'//g;
    $line =~ s/\\N'[^']*'//g;
    $line =~ s/\\[zu]\b?//g;
    $line =~ s/\\-/-/g;
    $line =~ s/\\e/\\/g;
    $line =~ s/\\ / /g;
    $line =~ s/\\\(..//g;          # any remaining \(xx
    $line =~ s/\\s[+-]?\d+//g;     # \s+n size
    $line =~ s/\\&//g;
    $line =~ s/\\\\//g;            # \\ -> \
    $line =~ s/[ \t]+/ /g;
    $line =~ s/^ //;
    $line =~ s/ $//;

    if ($line eq '') {
        print "\n" unless $prev_blank;
        $prev_blank = 1;
        next;
    }

    # keep only prose-like lines: mostly letters, almost no symbols, real words
    my $len     = length($line);
    my $letters = ($line =~ tr/A-Za-z//);
    my $symbols = ($line =~ tr/\\{}[]<>;=$|`~^//);
    my $words   = 0;
    $words++ while $line =~ /\b[A-Za-z]{2,}\b/g;
    next if $len < 25 || $words < 4 || $letters < 0.65 * $len || $symbols > 1;

    print $line, "\n";
    $prev_blank = 0;
}