Plugin Directory

Changeset 610523


Ignore:
Timestamp:
10/10/2012 10:17:38 PM (13 years ago)
Author:
ekeller
Message:
  • Custom fields are now supported
  • Large database can be indexed partially
Location:
opensearchserver-search/trunk
Files:
6 edited

Legend:

Unmodified
Added
Removed
  • opensearchserver-search/trunk/lib/oss_abstract.class.php

    r610356 r610523  
    140140    curl_setopt($rcurl, CURLOPT_BINARYTRANSFER, TRUE);
    141141    curl_setopt($rcurl, CURLOPT_RETURNTRANSFER, TRUE);
    142     curl_setopt($rcurl, CURLOPT_FOLLOWLOCATION, TRUE);
     142    curl_setopt($rcurl, CURLOPT_FOLLOWLOCATION, FALSE);
    143143    curl_setopt($rcurl, CURLOPT_MAXREDIRS, 16);
    144144    curl_setopt($rcurl, CURLOPT_VERBOSE, FALSE);
  • opensearchserver-search/trunk/lib/oss_results.class.php

    r603172 r610523  
    185185    $spellCheckWord = isset($fieldName)? $this->result->xpath('spellcheck/field[@name="' . $fieldName . '"]/word'):NULL;
    186186    $queryExact = '';
    187     foreach ($spellCheckWord as $each) {
    188       $queryExact .= $each[0]->suggest.' ';
     187    if (isset($spellCheckWord) && $spellCheckWord != null) {
     188      foreach ($spellCheckWord as $each) {
     189        $queryExact .= $each[0]->suggest.' ';
     190      }
    189191    }
    190192    return $queryExact;
  • opensearchserver-search/trunk/lib/oss_search.class.php

    r610356 r610523  
    3333/**
    3434 * @package OpenSearchServer
    35  */
     35*/
    3636class OssSearch extends OssSearchAbstract {
    3737
     
    6161    $this->filter  = array();
    6262    $this->sort    = array();
    63     $this->facets  = array();
     63    $this->facet  = array();
    6464    $this->query = NULL;
    6565    $this->lang = NULL;
     
    152152    return $this;
    153153  }
    154  
     154
    155155  /**
    156156   * @return OssSearch
    157157   */
    158158  public function collapseType($type) {
    159     $this->collapse['type'] = $type;
    160     return $this;
    161   }
    162  
     159    $this->collapse['type'] = $type;
     160    return $this;
     161  }
     162
    163163
    164164  /**
     
    225225    // Facets
    226226    foreach ((array)$this->facet as $field => $options) {
    227         if($options['multi']) {
    228             $facet = 'facet.multi=';
    229         }
    230         else if($options['multi_collapse']) {
    231             $facet = 'facet.multi.collapse=';
    232         }else {
    233             $facet = 'facet=';
    234         }
    235         $facet .= $field;
    236         if ($options['min'] !== NULL) {
    237             $facet .= '(' . $options['min'] . ')';
    238         }
    239         $queryChunks[] = $facet;
     227      if ($options['multi']) {
     228        $facet = 'facet.multi=';
     229      } else if ($options['multi_collapse']) {
     230        $facet = 'facet.multi.collapse=';
     231      } else {
     232        $facet = 'facet=';
     233      }
     234      $facet .= $field;
     235      if ($options['min'] !== NULL) {
     236        $facet .= '(' . $options['min'] . ')';
     237      }
     238      $queryChunks[] = $facet;
    240239    }
    241240
     
    245244    }
    246245    if ($this->collapse['type']) {
    247         $queryChunks[] = 'collapse.type=' . $this->collapse['type'];
     246      $queryChunks[] = 'collapse.type=' . $this->collapse['type'];
    248247    }
    249248    if ($this->collapse['mode'] !== NULL) {
  • opensearchserver-search/trunk/opensearchserver_admin.php

    r603172 r610523  
    2626function opensearchserver_create_index() {
    2727  $indexName = get_option('oss_indexname');
     28  $custom_fields = get_option('oss_custom_field');
    2829  $oss_api = opensearchserver_getapi_instance();
    2930  $index_list = $oss_api->indexList();
     
    3435
    3536  $oss_api->createIndex($indexName);
    36   opensearchserver_create_schema();
    37   opensearchserver_query_template();
     37  opensearchserver_create_schema($custom_fields);
     38  opensearchserver_query_template($custom_fields);
    3839  opensearchserver_spellcheck_query_template();
    3940  return TRUE;
     
    4950}
    5051
    51 
    5252/*
    5353 * Function create schema values for OpenSearchServer
    5454*/
    5555
    56 function opensearchserver_create_schema() {
     56function opensearchserver_create_schema($custom_fields) {
    5757  $schema = opensearchserver_getschema_instance();
    5858  $schema_xml = $schema->getSchema();
     
    6767    opensearchserver_setField($schema,$schema_xml,'contentPhonetic','PhoneticAnalyzer','yes','yes','positions_offsets','no','no');
    6868  }
    69   opensearchserver_setField($schema,$schema_xml,'content','TextAnalyzer','compress','yes','positions_offsets','yes','no');
     69  opensearchserver_setField($schema,$schema_xml,'content','TextAnalyzer','yes','yes','positions_offsets','yes','no');
    7070  opensearchserver_setField($schema,$schema_xml,'contentExact','StandardAnalyzer','no','yes','no','no','no');
    7171  opensearchserver_setField($schema,$schema_xml,'timestamp',NULL,'no','yes','yes','no','no');
     
    7676  opensearchserver_setField($schema,$schema_xml,'categories','TextAnalyzer','yes','yes','no','yes','no');
    7777  opensearchserver_setField($schema,$schema_xml,'categoriesExact',NULL,'yes','yes','no','yes','no');
     78  if (isset($custom_fields) && $custom_fields != null) {
     79    $custom_fields_array = explode(',', $custom_fields);
     80    foreach ($custom_fields_array as $field) {
     81      $field = opensearchserver_clean_field($field);
     82      if (strlen($field) > 0) {
     83        opensearchserver_setField($schema,$schema_xml,'custom_'.$field,NULL,'yes','yes','no','yes','no');
     84      }
     85    }
     86  }
    7887}
    7988
     
    94103 * Function to update the query template
    95104*/
    96 function opensearchserver_query_template() {
     105function opensearchserver_query_template($custom_fields) {
    97106  $query_template = opensearchserver_getsearchtemplate_instance();
    98107  $oss_query = stripcslashes(get_option('oss_query'));
     
    104113    $query_template->setSnippetField('search','contentPhonetic', 300, 'b', NULL, 'SentenceFragmenter');
    105114  }
    106   $query_template->setReturnField('search',"url");
    107   $query_template->setReturnField('search',"user_url");
    108   $query_template->setReturnField('search',"type");
    109   $query_template->setReturnField('search',"user_name");
    110   $query_template->setReturnField('search',"user_email");
    111   $query_template->setReturnField('search',"categories");
     115  $query_template->setReturnField('search','url');
     116  $query_template->setReturnField('search','user_url');
     117  $query_template->setReturnField('search','type');
     118  $query_template->setReturnField('search','user_name');
     119  $query_template->setReturnField('search','user_email');
     120  $query_template->setReturnField('search','categories');
     121  if (isset($custom_fields) && $custom_fields != null) {
     122    $custom_fields_array = explode(',', $custom_fields);
     123    foreach ($custom_fields_array as $field) {
     124      $field = opensearchserver_clean_field($field);
     125      if (strlen($field) > 0) {
     126        $query_template->setReturnField('search','custom_'.$field);
     127      }
     128    }
     129  }
    112130}
    113131
     
    124142}
    125143
     144/**
     145 * Check if the document list should be indexed
     146 * @param OSSIndexDocument $index
     147 * @param number $limit
     148 */
     149function opensearchserver_checkindex(OSSIndexDocument $index, $limit = 1, $idx = 0, $total = 0) {
     150  global $wpdb;
     151  if ($index->count() < $limit) {
     152    return $index;
     153  }
     154  opensearchserver_start_indexing($index);
     155  $index = null;
     156  wp_cache_flush();
     157  $wpdb->flush();
     158  if (function_exists('gc_enabled')) {
     159    if (gc_enabled()) {
     160      gc_collect_cycles();
     161    }
     162  }
     163
     164  if ($idx != 0 && $total != 0) {
     165    $percent = (floatval($idx) / floatval($total)) * 100;
     166    $mem = floatval(memory_get_usage()) / 1024 / 1024;
     167    opensearchserver_display_messages(sprintf('Completed: %.2f - Memory usage: %2f', $percent, $mem));
     168  }
     169  return new OSSIndexDocument();
     170}
     171
    126172/*
    127173 * Function to reindex the website.
    128174*/
    129 function opensearchserver_reindex_site($id,$type) {
     175function opensearchserver_reindex_site($id,$type, $from = 0, $to = 0) {
    130176  global $wpdb;
    131177  $oss_server_url = get_option('oss_serverurl');
     
    141187  $ossEngineConnectTimeOut = config_request_value('ossEngineConnectTimeOut', 5, 'engineConnectTimeOut');
    142188  $ossEngineIndex = config_request_value('ossEngineIndex', $oss_indexname, 'engineIndex');
    143   if($id)   {
     189  if($id) {
    144190    $delete='id:'.$type.'_'.$id;
    145191    opensearchserver_delete_document($delete);
    146     $sql_suffix = 'FROM '.$table_name_posts.' p LEFT JOIN  '.$table_name_users.' u ON p.post_author = u.ID WHERE `post_status` = \'publish\' AND p.ID ='.$id;
    147     $sql_query = 'SELECT p.ID,post_type,post_title,post_content,guid,post_date_gmt,post_author,user_nicename,user_url,user_email '.$sql_suffix;
    148     $sql_posts = $wpdb->get_results($sql_query);
    149     $index = opensearchserver_add_documents_to_index($lang, $sql_posts, $custom_fields);
    150     opensearchserver_start_indexing($index);
    151   }else {
    152     opensearchserver_delete_document('*:*');
    153     wp_cache_flush();
    154     $batch = 200;
    155     $sql_suffix = 'FROM '.$table_name_posts.' p LEFT JOIN  '.$table_name_users.' u ON p.post_author = u.ID WHERE `post_status` = \'publish\'';
    156     $total_count = $wpdb->get_var($wpdb->prepare( 'SELECT COUNT(*) '.$sql_suffix));
    157     $wpdb->flush();
    158     $sql_query = 'SELECT p.ID,post_type,post_title,post_content,guid,post_date_gmt,post_author,user_nicename,user_url,user_email '.$sql_suffix;
    159 
    160     $current_pos = 0;
    161     while ($current_pos < $total_count) {
    162       $row_fetch = $total_count - $current_pos;
    163       if ($row_fetch > $batch) {
    164         $row_fetch = $batch;
    165       }
    166 
    167       $sql_posts = $wpdb->get_results($sql_query.' LIMIT '.$current_pos.','.$batch);
    168       $index = opensearchserver_add_documents_to_index($lang, $sql_posts, $custom_fields);
    169       opensearchserver_start_indexing($index);
    170       $wpdb->flush();
    171       wp_cache_flush();
    172       unset($sql_posts);
    173       unset($index);
    174       $current_pos += $row_fetch;
    175     }
     192    $index = new OSSIndexDocument();
     193    opensearchserver_add_documents_to_index($index, $lang, get_post($id), $custom_fields);
     194    opensearchserver_checkindex($index);
     195  } else {
     196    $from = (int) $from;
     197    $to = (int) $to;
     198    if ($from == 0 && $to == 0) {
     199      opensearchserver_delete_document('*:*');
     200    }
     201    $limitSuffix = $to != 0 ? ' LIMIT '.$from.','.($to - $from) : '';
     202    $sql_query = 'SELECT ID FROM '.$wpdb->posts.' WHERE post_status = \'publish\' ORDER BY ID'.$limitSuffix;
     203    $posts = $wpdb->get_results($sql_query);
     204    $total_count = count($posts);
     205    $index = new OSSIndexDocument();
     206    for ($i = 0; $i < $total_count; $i++) {
     207      $id = $posts[$i]->ID;
     208      opensearchserver_add_documents_to_index($index, $lang, get_post($id), $custom_fields);
     209      $index = opensearchserver_checkindex($index, 200, $i, $total_count);
     210    }
     211    opensearchserver_checkindex($index, 1, $i, $total_count);
    176212  }
    177213  opensearchserver_optimize();
    178214  opensearchserver_autocompletionBuild();
    179   $index_status=1;
    180   return $index_status;
     215  return 1;
    181216}
    182217
     
    245280}
    246281
    247 function opensearchserver_add_documents_to_index($lang, $sql_posts, $customFields) {
    248   $index = new OSSIndexDocument();
    249   $lang= substr(get_locale(), 0, 2);
    250   foreach($sql_posts as $post){
    251     $content=$post->post_content;
    252     $content = apply_filters('the_content', $content);
    253     $content = str_replace(']]>', ']]&gt;', $content);
    254     $content = opensearchserver_encode(strip_tags($content));
    255     $document = $index->newDocument($lang);
    256     $document->newField('id', $post->post_type.'_'.$post->ID);
    257     $document->newField('type', strip_tags($post->post_type));
    258     $title = opensearchserver_stripInvalidXml(strip_tags($post->post_title));
    259     $document->newField('title', $title);
    260     $document->newField('titleExact', $title);
    261     $document->newField('titlePhonetic', $title);
    262     $content = opensearchserver_stripInvalidXml($content);
    263     $document->newField('content', $content);
    264     $document->newField('contentExact', $content);
    265     $document->newField('contentPhonetic', $content);
    266     $document->newField('url', get_permalink($post->ID));
    267     $document->newField('urlExact', get_permalink($post->ID));
    268     $document->newField('timestamp', $post->post_date_gmt);
    269     $document->newField('user_name', $post->user_nicename);
    270     $document->newField('user_email', $post->user_email);
    271     $document->newField('user_email', $post->user_url);
    272     $categories_data= '';
    273 
    274     // Handling categories
    275     $categories_data = null;
    276     $categories = get_the_category($post->ID);
    277     if ( ! $categories == NULL ) {
    278       foreach( $categories as $category ) {
    279         $categories_data .= $category->cat_name.' ';
    280         $document->newField('categories', $category->cat_name);
    281         $document->newField('categoriesExact', $category->cat_name);
     282function opensearchserver_add_documents_to_index(OSSIndexDocument $index, $lang, $post, $customFields) {
     283  $user = get_userdata($post->post_author);
     284  $content = $post->post_content;
     285  $content = apply_filters('the_content', $content);
     286  $content = str_replace(']]>', ']]&gt;', $content);
     287  $content = opensearchserver_encode(strip_tags($content));
     288  $content = opensearchserver_stripInvalidXml($content);
     289  $document = $index->newDocument($lang);
     290  $document->newField('id', $post->post_type.'_'.$post->ID);
     291  $document->newField('type', strip_tags($post->post_type));
     292  $title = opensearchserver_stripInvalidXml(strip_tags($post->post_title));
     293  $document->newField('title', $title);
     294  $document->newField('titleExact', $title);
     295  $document->newField('titlePhonetic', $title);
     296  $document->newField('content', $content);
     297  $document->newField('contentExact', $content);
     298  $document->newField('contentPhonetic', $content);
     299  $document->newField('url', get_permalink($post->ID));
     300  $document->newField('urlExact', get_permalink($post->ID));
     301  $document->newField('timestamp', $post->post_date_gmt);
     302  $document->newField('user_name', $user->user_nicename);
     303  $document->newField('user_email', $user->user_email);
     304  $document->newField('user_email', $user->user_url);
     305  $categories_data= '';
     306
     307  // Handling categories
     308  $categories_data = null;
     309  $categories = get_the_category($post->ID);
     310  if ($categories != NULL) {
     311    foreach( $categories as $category ) {
     312      $categories_data .= $category->cat_name.' ';
     313      $document->newField('categories', $category->cat_name);
     314      $document->newField('categoriesExact', $category->cat_name);
     315    }
     316    $categories = null;
     317  }
     318
     319  // Handling custom fields
     320  $custom_clean_all='';
     321  if($customFields) {
     322    $custom_fields_array = explode(',',$customFields);
     323    foreach ($custom_fields_array as $field) {
     324      $field = trim($field);
     325      $custom_content = '';
     326      $custom_values=get_post_custom_values($field, $post->ID);
     327      if(is_array($custom_values)) {
     328        foreach ($custom_values as $values) {
     329          $custom_content .= $values.' ';
     330        }
     331      }else {
     332        $custom_content = $custom_values;
    282333      }
    283     }
    284 
    285     // Handling custom fields
    286     $custom_clean_all='';
    287     if($customFields) {
    288       $custom_fields_array=explode(",",$customFields);
    289       foreach ($custom_fields_array as $field) {
    290         $custom_content = '';
    291         $custom_values=get_post_custom_values($field,$post->ID);
    292         if(is_array($custom_values)) {
    293           foreach ($custom_values as $values) {
    294             $custom_content .= $values." ";
    295           }
    296         }else {
    297           $custom_content = $custom_values;
    298         }
    299         $content_br = nl2br($custom_content);
    300         $content_clean=str_replace("<br />", ' ', $content_br);
    301         $document->newField("custom_".clean_field($field), stripInvalidXml(strip_tags($content_clean)));
    302         $custom_clean_all .=' '.$content_clean;
    303       }
    304     }
    305     // Build all content field
    306     $all_content = opensearchserver_stripInvalidXml(strip_tags($post->post_title)). ' '.$content;
    307     if ($categories_data) {
    308       $all_content = strip_tags($all_content.' '.$categories_data);
    309     }
    310     if ($custom_clean_all) {
    311       $all_content .= ' ' .$custom_clean_all;
    312     }
    313     $document->newField("allContent", strip_tags($all_content.' '.$categories_data));
    314   }
    315   return $index;
     334      $content_br = nl2br($custom_content);
     335      $content_clean=str_replace('<br />', ' ', $content_br);
     336      $document->newField('custom_'.opensearchserver_clean_field($field), opensearchserver_stripInvalidXml(strip_tags($content_clean)));
     337      $custom_clean_all .=' '.$content_clean;
     338    }
     339    $custom_fields_array = null;
     340  }
     341  // Build all content field
     342  $all_content = opensearchserver_stripInvalidXml(strip_tags($post->post_title)). ' '.$content;
     343  if ($categories_data) {
     344    $all_content = strip_tags($all_content.' '.$categories_data);
     345  }
     346  if ($custom_clean_all) {
     347    $all_content .= ' ' .$custom_clean_all;
     348    $custom_clean_all = null;
     349  }
     350  $document->newField("allContent", strip_tags($all_content.' '.$categories_data));
    316351}
    317352
     
    389424    update_option('oss_key', $oss_key);
    390425    opensearchserver_display_messages('OpenSearchServer Settings has been updated');
    391 
    392   }if($action == 'query_settings') {
     426  }
     427  if($action == 'query_settings') {
    393428    $delete = isset($_POST['oss_delete']) ? $_POST['oss_delete'] :NULL;
    394429    $delete_action = isset($_POST['opensearchserver_delete']) ? $_POST['opensearchserver_delete'] :NULL;
     
    433468    }
    434469
    435   } if($action == 'index_settings') {
     470  }
     471  if($action == 'index_settings') {
    436472    $is_index_created = opensearchserver_create_index();
    437473    opensearchserver_display_messages('Index '.get_option('oss_indexname').' Created successfully');
    438474     
    439   }if($action == 'custom_field_settings') {
     475  }
     476  if($action == 'custom_field_settings') {
    440477    $oss_custom_field = isset($_POST['oss_custom_field']) ? $_POST['oss_custom_field'] :NULL;
    441478    update_option('oss_custom_field', $oss_custom_field);
     
    443480  }
    444481  if($action == 'opensearchserver_reindex') {
    445     $is_index_created = opensearchserver_create_index();
    446     $index_success = opensearchserver_reindex_site(NULL,NULL);
     482    $oss_index_from = isset($_POST['oss_index_from']) ? $_POST['oss_index_from'] : NULL;
     483    $oss_index_to = isset($_POST['oss_index_to']) ? $_POST['oss_index_to'] : NULL;
     484    update_option('oss_index_from', $oss_index_from);
     485    update_option('oss_index_to', $oss_index_to);
     486    $index_success = opensearchserver_reindex_site(NULL,NULL, $oss_index_from, $oss_index_to);
    447487    opensearchserver_display_messages('Re indexing has been finished successfully.');
    448488  }
     
    546586                                </thead>
    547587                                <tbody>
    548 
    549588                                    <?php
    550589                                    foreach($facets as $facet) {
     
    552591                                    <tr>
    553592                                        <td><?php print $fields[$facet]; ?></td>
    554                                         <input type="hidden" name="oss_delete" id="oss_submit"
    555                                             value="<?php print $facet; ?>" />
    556                                         <td><input type="submit" name="opensearchserver_delete"
    557                                             value="Delete" class="button-secondary" /></td>
     593                                        <td><input type="hidden" name="oss_delete" id="oss_submit"
     594                                            value="<?php print $facet; ?>" /> <input type="submit"
     595                                            name="opensearchserver_delete" value="Delete"
     596                                            class="button-secondary" /></td>
    558597                                    </tr>
    559598                                    <?php }?>
    560 
    561599                                </tbody>
    562600                            </table>
     
    683721                <form id="reindex_settings" name="reindex_settings" method="post"
    684722                    action="">
    685                     <input type="hidden" name="oss_submit" id="oss_submit"
    686                         value="opensearchserver_reindex" /> <input type="submit"
    687                         name="opensearchserver_submit" value="Synchronize / Re-Index"
    688                         class="button-primary" />
     723                    <p>
     724                        <label for="oss_index_from">From index</label>:<br /> <input
     725                            type="text" name="oss_index_from" id="oss_index_from" size="15"
     726                            value="<?php print get_option('oss_index_from');?>" /> <br />
     727                    </p>
     728                    <p>
     729                        <label for="oss_index_to">To index</label>:<br /> <input
     730                            type="text" name="oss_index_to" id="oss_index_to" size="15"
     731                            value="<?php print get_option('oss_index_to');?>" /> <br />
     732                    </p>
     733                    <p>
     734                        <input type="hidden" name="oss_submit" id="oss_submit"
     735                            value="opensearchserver_reindex" /> <input type="submit"
     736                            name="opensearchserver_submit" value="Synchronize / Re-Index"
     737                            class="button-primary" />
     738                    </p>
    689739                </form>
    690740            </div>
  • opensearchserver-search/trunk/opensearchserver_search_functions.php

    r610356 r610523  
    55
    66function opensearchserver_getresult_instance($result) {
     7  if ($result == null) {
     8    return null;
     9  }
    710  return new OssResults($result);
    811}
     
    1316
    1417function opensearchserver_getspellcheck($result) {
     18  if ($result == null)
     19    return null;
    1520  $spell_field = get_option('oss_spell').'Exact';
    1621  return opensearchserver_getresult_instance($result)->getSpellSuggest($spell_field);
     
    7984        }
    8085      }
    81       $result = $search->query($query)->template('search')->execute(5);
     86      $result = $search->query($query)->template('search')->execute();
    8287    }else {
    83       $result = $search->query($query)->template('spellcheck')->execute(5);
     88      $result = $search->query($query)->template('spellcheck')->execute();
    8489    }
    8590    return $result;
     
    99104function opensearchserver_add_facets_search($search) {
    100105  $facets = get_option('oss_facet');
    101   if (isset($facet) && $facet != null) {
     106  if (isset($facets) && $facets != null) {
    102107    foreach ($facets as $facet) {
    103       $search->facet($facet,1);
     108      $search->facet($facet, 1);
    104109    }
    105110  }
  • opensearchserver-search/trunk/template/opensearchserver_search.php

    r610356 r610523  
    2424    $oss_result = opensearchserver_getsearchresult($query, FALSE, TRUE);
    2525    $oss_result_facet = opensearchserver_getsearchresult($query, FALSE, FALSE);
    26     $oss_sp= isset($_REQUEST['sp']) ? $_REQUEST['sp'] :NULL;
     26    $oss_sp = isset($_REQUEST['sp']) ? $_REQUEST['sp'] :NULL;
    2727    if (isset($oss_result) && $oss_result instanceof SimpleXMLElement && isset($oss_result_facet) && $oss_result_facet instanceof SimpleXMLElement) {
    2828      $oss_results = opensearchserver_getresult_instance($oss_result);
     
    5555    <div id="oss-filter">
    5656        <?php $facets = get_option('oss_facet');
    57         if (isset($facet) && $facet != null) {
     57        if (isset($facets) && $facets != null) {
    5858        foreach ($facets as $facet) {
    5959          $facet_results = $oss_result_facets->getFacet($facet);
Note: See TracChangeset for help on using the changeset viewer.