/* quickstartindex.cc: Simplest possible indexer */

#include <xapian.h>
#include <iostream>
#include <string>
#include <vector>
#include <cctype>
#include <algorithm>

using namespace std;

#define MAX_KEY 230

#define SPLIT_TITLE_INTO_KEYWORDS

// Split a string into its tokens, based on the given delimiters
void Tokenize(const string& str, vector<string>& tokens, 
              const string& delimiters)
{
    // Skip delimiters at beginning.
    string::size_type lastPos = str.find_first_not_of(delimiters, 0);
    // Find first "non-delimiter".
    string::size_type pos     = str.find_first_of(delimiters, lastPos);

    while (string::npos != pos || string::npos != lastPos)
    {
        // Found a token, add it to the vector.
        tokens.push_back(str.substr(lastPos, pos - lastPos));
        // Skip delimiters.  Note the "not_of"
        lastPos = str.find_first_not_of(delimiters, pos);
        // Find next "non-delimiter"
        pos = str.find_first_of(delimiters, lastPos);
    }
}

// We will perform case insensitive searches, 
// so we need a function to lowcase() a string
char to_lower (const char c) { return tolower(c); }
void lowcase(string& s) { transform(s.begin(), s.end(), s.begin(), to_lower); }

int main(int argc, char **argv)
{
    unsigned total = 0;
    try {
        // Make the database
        Xapian::WritableDatabase database("db/", Xapian::DB_CREATE_OR_OPEN);

        string docId;
        while(1) {
            string title;
            if (cin.eof()) break;
            getline(cin, title);
            int l = title.length();
            if (l>4 && title[0] == '#' && title.substr(l-4, 4) == ".bz2") {
                docId = title.substr(1, string::npos);  
                continue;
            }

            string Title = title;
            lowcase(title);

            // Make the document
            Xapian::Document newdocument;

            // Target: filename and the exact title used
            string target = docId + string(":") + Title;
            if (target.length()>MAX_KEY)
                target = target.substr(0, MAX_KEY);
            newdocument.set_data(target);

            // 1st Source: the lowercased title
            if (title.length() > MAX_KEY)
                title = title.substr(0, MAX_KEY);
            newdocument.add_posting(title.c_str(), 1);

            vector<string> keywords;
            Tokenize(title, keywords, " ");

            // 2nd source: All the title's lowercased words
            int cnt = 2;
            for (vector<string>::iterator it=keywords.begin(); 
                it!=keywords.end(); it++) 
            {
                if (it->length() > MAX_KEY)
                    *it = it->substr(0, MAX_KEY);
                newdocument.add_posting(it->c_str(), cnt++);
            }

            try {
                //cout << "Added " << title << endl;
                // Add the document to the database
                database.add_document(newdocument);
            } catch(const Xapian::Error &error) {
                cout << "Exception: "  << error.get_msg();
                cout << "\nWhen adding:\n" << title;
                cout << "\nOf length " << title.length() << endl;
            }
            total ++;

            if ((total % 8192) == 0) {
                cout << total << " articles indexed so far" << endl;
            }
        }
    } catch(const Xapian::Error &error) {
        cout << "Exception: "  << error.get_msg() << endl;
    }
    cout << total << " articles indexed." << endl;
}