fulltext._pike

Go to the documentation of this file.

/*
 * create temporary table ft_id (ob_id int primary key, count int);
 * insert into ft_id select ob_id, 0 from ob_data where ob_attr='DOC_MIME_TYPE' and ob_data='"text/html"';
 * replace into ft_id select ob_id, 1 from doc_ft;
 * delete from ft_id where count = 1;
 */
class fulltext {
public:

protected:
 void check_ft_integrity()
{
    Sql.Sql handle = Sql.Sql(serverCfg["database"]);
    handle->query("create temporary table ft_id "+
                  "(ob_id int primary key, count int)");
    handle->query("insert into ft_id select distinct "+
                  "ob_id,0 from ob_data where ob_attr='DOC_MIME_TYPE'"+
                  "and ob_data='\""+
                  indices(mStripFilter)*"\"' or ob_data='\""+"\"'");
    handle->query("replace into ft_id select distinct ob_id, 1 from doc_ft");
    handle->query("delete from ft_id where count =1");
    array missing = handle->query("select distinct ob_id from ft_id");
    handle->query("drop table ft_id");
    foreach (missing, mixed a)
    {
        object o = connection->find_object((int)a["ob_id"]);
        if (objectp(o))
            oQueue->write(Doc(o));
    }
}

public:

private:
 private void got_kill(int sig)
{
    _exit(1);  
}

public:

int main(int argc, array argv)
{
    init( "fulltext", argv + ({ "--eid="+EVENT_UPLOAD }) );

    if (catch{mStripFilter = read_config_file(CONFIG_DIR+"/services/fulltext.cfg");})
      mStripFilter = ([ "text/html" : "html2text" ]);

    if ( !mStripFilter["text/plain"]) 
      mStripFilter["text/plain"] = "";

    // check strip filters and remove any that don't work:
    foreach ( indices(mStripFilter), string mime ) {
      mixed executable = mStripFilter[ mime ];
      if ( !stringp(mime) || !stringp(executable) || executable == "" )
        continue;
      mixed err = catch {
        Process.create_process( ({ executable }), ([ ]) )->wait();
      };
      if ( err ) {
        werror( "[%s] Executable for %s strip filter not working: %s (Error: %s)\n",
                timelib.log_time(), mime, executable, err[0]-"\n" );
        m_delete( mStripFilter, mime );
      }
    }

    signal(signum("QUIT"), got_kill);
    
    mixed err = catch{
        create_table(serverCfg["database"]);
        start();
        persistence_module = send_cmd( 0, "get_module", "persistence" );
        content_in_database = send_cmd( persistence_module, "get_store_content_in_database", 0 );
        content_in_filesystem = send_cmd( persistence_module, "get_store_content_in_filesystem", 0 );
    };

    tStripDemon = thread_create(strip_demon);
    thread_create(check_ft_integrity);

    if (err)
        werror("Startup of fulltext service failed.\n"+
               master()->describe_backtrace(err)+"\n");
    return -17;
}

mapping read_config_file(string fname)
{
    Parser.XML.Tree.Node node = Parser.XML.Tree.parse_file(fname);
    if (!objectp(node))
        error("Failed to parse config file %s\n", fname);

    mapping data = ([]);
    node = node->get_first_element("config");
    foreach(node->get_elements(), Parser.XML.Tree.Node n)
    {
   mapping attributes = n->get_attributes();
   switch ( n->get_tag_name() ) {
   case "doc_strip":
       if ( !stringp(attributes->mime) ) 
           error("Missing mime attribute for doc_strip in config file!");
       data[attributes["mime"]] = n->get_last_child()->get_text();
       break;
   case "limits":
       foreach(n->get_elements(), Parser.XML.Tree.Node l) {
           int lsz = (int)l->get_last_child()->get_text();
           werror( "[%s] Using Limit: " + l->get_tag_name() + " = "+ lsz +"\n",
                        timelib.log_time() );
           mLimits[l->get_tag_name()] = lsz;
       }
       break;
   default:
            werror( "[%s] Unknown Tag in Config file : " + n->get_tag_name() +"\n",
                    timelib.log_time() );
   }
    }
    return data;
}


};