class.SearchUtil.php

Go to the documentation of this file.

<?php
/**
 * wCMF - wemove Content Management Framework
 * Copyright (C) 2005-2014 wemove digital solutions GmbH
 *
 * Licensed under the terms of any of the following licenses
 * at your choice:
 *
 * - GNU Lesser General Public License (LGPL)
 *   http://www.gnu.org/licenses/lgpl.html
 * - Eclipse Public License (EPL)
 *   http://www.eclipse.org/org/documents/epl-v10.php
 *
 * See the license.txt file distributed with this work for
 * additional information.
 *
 * $Id: class.SearchUtil.php 1462 2014-02-04 23:52:27Z iherwig $
 */
set_include_path(get_include_path().PATH_SEPARATOR.BASE.'wcmf/3rdparty/zend');

require_once BASE.'wcmf/3rdparty/zend/Zend/Search/Lucene.php';
require_once BASE.'wcmf/3rdparty/zend/Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php';
require_once BASE.'wcmf/lib/util/class.InifileParser.php';
require_once BASE.'wcmf/lib/util/class.StringUtil.php';

/**
 * @class SearchUtil
 * @ingroup Util
 * @brief This class provides access to the search based on Zend_Search_Lucene.
 * The search index stored in the location that is defined by the configuration key 'indexPath'
 * in the configuration section 'search'. To manage PersistentObjects in the index use the
 * methods SearchUtil::indexInSearch() and SearchIndex::deleteFromSearch() and SearchUtil::commitIndex().
 * The method SearchUtil::getIndex() offers direct access to the search index for advanced operations.
 *
 * @author Niko <enikao@users.sourceforge.net>
 */
class SearchUtil
{
  const INI_SECTION = 'search';
  const INI_INDEX_PATH = 'indexPath';

  private static $isActivated = null;
  private static $index;
  private static $indexPath;
  private static $indexIsDirty = false;

  /**
   * Get the search index.
   * @param create Boolean whether to create the index, if it does not exist [default: true]
   * @return An instance of Zend_Search_Lucene_Interface or null
   */
  public static function getIndex($create = true)
  {
    if (!self::isActivated()) {
      return null;
    }

    if (!self::$index || $create)
    {
      $indexPath = self::getIndexPath();

      $analyzer = new Analyzer();

      // add stop words filter
      $stopWords = self::getStopWords();
      $stopWordsFilter = new Zend_Search_Lucene_Analysis_TokenFilter_StopWords($stopWords);
      $analyzer->addFilter($stopWordsFilter);

      Zend_Search_Lucene_Analysis_Analyzer::setDefault($analyzer);
      Zend_Search_Lucene_Search_Query_Wildcard::setMinPrefixLength(0);
      Zend_Search_Lucene_Search_QueryParser::setDefaultEncoding('UTF-8');
      Zend_Search_Lucene_Search_QueryParser::setDefaultOperator(Zend_Search_Lucene_Search_QueryParser::B_AND);

      try {
        self::$index = Zend_Search_Lucene::open($indexPath);
        //self::$index->setMaxMergeDocs(5);
        //self::$index->setMergeFactor(5);
      }
      catch (Zend_Search_Lucene_Exception $ex) {
        self::$index = self::resetIndex();
      }
    }
    return self::$index;
  }

  /**
   * Reset the search index.
   */
  public static function resetIndex()
  {
    if (!self::isActivated()) {
      return;
    }
    $indexPath = self::getIndexPath();
    return Zend_Search_Lucene::create($indexPath);
  }

  /**
   * Add a PersistentObject instance to the search index. This method modifies the
   * index. For that reason SearchUtil::commitIndex() should be called afterwards.
   * @param obj The PersistentObject instance.
   */
  public static function indexInSearch(&$obj)
  {
    if (!self::isActivated()) {
      return;
    }
    if ($obj->isIndexInSearch()) {
      $index = self::getIndex();

      $doc = new Zend_Search_Lucene_Document();

      $valueNames = $obj->getValueNames(DATATYPE_ATTRIBUTE);

      $doc->addField(Zend_Search_Lucene_Field::unIndexed('oid', $obj->getOID(), 'UTF-8'));
      $typeField = Zend_Search_Lucene_Field::keyword('type', $obj->getType(), 'UTF-8');
      $typeField->isStored = false;
      $doc->addField($typeField);

      foreach ($valueNames as $curValueName) {
        $properties = $obj->getValueProperties($curValueName);
        $inputType = $properties['input_type'];
        $value = self::encodeValue($obj->getValue($curValueName, DATATYPE_ATTRIBUTE), $inputType);
        if (preg_match('/^text|^f?ckeditor/', $inputType)) {
          $value = strip_tags($value);
          $doc->addField(Zend_Search_Lucene_Field::unStored($curValueName, $value, 'UTF-8'));
        }
        else {
          $field = Zend_Search_Lucene_Field::keyword($curValueName, $value, 'UTF-8');
          $field->isStored = false;
          $doc->addField($field);
        }
      }

      $term = new Zend_Search_Lucene_Index_Term($obj->getOID(), 'oid');
      $docIds  = $index->termDocs($term);
      foreach ($docIds as $id) {
        $index->delete($id);
      }

      $index->addDocument($doc);
      self::$indexIsDirty = true;
    }
  }

  private static function encodeValue($value, $inputType)
  {
    if (preg_match('/^f?ckeditor/', $inputType)) {
      $value = html_entity_decode($value, ENT_QUOTES, 'UTF-8');
    }
    return trim($value);
  }

  /**
   * Delete a PersistentObject instance from the search index.
   * @param obj The PersistentObject instance.
   */
  public static function deleteFromSearch(&$obj)
  {
    if (!self::isActivated()) {
      return;
    }
    if ($obj->isIndexInSearch())
    {
      $index = self::getIndex();

      $term = new Zend_Search_Lucene_Index_Term($obj->getOID(), 'oid');
      $docIds  = $index->termDocs($term);
      foreach ($docIds as $id)
      {
        $index->delete($id);
      }
      self::$indexIsDirty = true;
    }
  }

  /**
   * Commit any changes made by using SearchUtil::indexInSearch() and SearchIndex::deleteFromSearch().
   * @note This method only commits the index if changes were made using the methods mentioned above.
   * @param optimize Boolean whether the index should be optimized after commit [default: true].
   */
  public static function commitIndex($optimize = true)
  {
    if (!self::isActivated()) {
      return;
    }
    if (self::$indexIsDirty)
    {
      $index = self::getIndex(false);
      if ($index) {
        $index->commit();
        if ($optimize) {
          $index->optimize();
        }
      }
    }
  }

  /**
   * Optimize the index
   */
  public static function optimizeIndex()
  {
    if (!self::isActivated()) {
      return;
    }
    $index = self::getIndex(false);
    if ($index) {
      $index->optimize();
    }
  }

  /**
   * Get the path to the index.
   * @return The path.
   */
  private static function getIndexPath()
  {
    if (!self::$indexPath)
    {
      $parser = InifileParser::getInstance();
      if (($path = $parser->getValue(self::INI_INDEX_PATH, self::INI_SECTION)) !== false)
      {
        self::$indexPath = BASE . 'application/' . $path;

        if (!file_exists(self::$indexPath)) {
          FileUtil::mkdirRec(self::$indexPath);
        }

        if (!is_writeable(self::$indexPath)) {
          Log::error("Index path '".self::$indexPath."' is not writeable.", __CLASS__);
        }

        Log::debug("Lucene index location: ".self::$indexPath, __CLASS__);
      }
      else
      {
        Log::error($parser->getErrorMsg(), __CLASS__);
      }
    }
    return self::$indexPath;
  }

  /**
   * Check if a index path is defined in the configuration.
   * @return Boolean
   */
  public static function isActivated()
  {
    if (self::$isActivated === null) {
      $parser = InifileParser::getInstance();
      self::$isActivated = $parser->getValue(self::INI_INDEX_PATH, self::INI_SECTION) !== false;
    }
    return self::$isActivated;
  }

  /**
   * Get a list of words that are forbidden to search for
   * @return Array
   */
  public static function getStopWords()
  {
    return explode("\n", $GLOBALS['STOP_WORDS']);
  }

  /**
   * Search for searchTerm in index
   * @param searchTerm
   * @param pagingInfo
   * @return Associative array with object ids as keys and
   * associative array with keys 'oid', 'score', 'summary' as value
   */
  public static function find($searchTerm, &$pagingInfo)
  {
    $results = array();
    if (!self::isActivated()) {
      return $results;
    }
    $index = self::getIndex(false);
    if ($index) {
      $persistenceFacade = PersistenceFacade::getInstance();
      $query = Zend_Search_Lucene_Search_QueryParser::parse($searchTerm, 'UTF-8');
      try {
        $hits = $index->find($query);
        if ($pagingInfo != null && $pagingInfo->getPageSize() > 0) {
          $pagingInfo->setTotalCount(sizeof($hits));
          $hits = array_slice($hits, $pagingInfo->getIndex(), $pagingInfo->getPageSize());
        }
        foreach($hits as $hit) {
          $oid = $hit->oid;

          // get the summary with highlighted text
          $summary = '';
          $highlightedRegex = '/((<b style="color:black;background-color:#[0-9a-f]{6}">)+)([^<]+?)((<\/b>)+)/';
          $obj = $persistenceFacade->load($oid, BUILDDEPTH_SINGLE);
          $valueNames = $obj->getValueNames(DATATYPE_ATTRIBUTE);
          foreach ($valueNames as $curValueName) {
            $properties = $obj->getValueProperties($curValueName);
            $inputType = $properties['input_type'];
            $value = self::encodeValue($obj->getValue($curValueName, DATATYPE_ATTRIBUTE), $inputType);
            if (strlen($value) > 0) {
              $highlighted = $query->htmlFragmentHighlightMatches(strip_tags($value), 'UTF-8');
              $matches = array();
              if (preg_match($highlightedRegex, $highlighted, $matches)) {
                $hit = $matches[3];
                $highlighted = preg_replace($highlightedRegex, ' <em class="highlighted">$3</em> ', $highlighted);
                $highlighted = trim(preg_replace('/&#13;|[\n\r\t]/', ' ', $highlighted));
                $excerpt = StringUtil::excerpt($highlighted, $hit, 300, '');
                $summary = $excerpt;
                break;
              }
            }
          }
          $results[$oid] = array(
              'oid' => $oid,
              'score' => $hit->score,
              'summary' => $summary
          );
        }
      }
      catch (Exception $ex) {
        // do nothing, return empty result
      }
    }
    return $results;
  }
}

class Analyzer extends Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive {
  /**
   * Override method to make sure we are using utf-8
   */
  public function setInput($data, $encoding = '')
  {
    parent::setInput($data, 'UTF-8');
  }
}

/**
 * Standard german/english stop words taken from Lucene's StopAnalyzer
 */
$GLOBALS['STOP_WORDS'] = <<<'EOD'
ein
einer
eine
eines
einem
einen
der
die
das
dass
daß
du
er
sie
es
was
wer
wie
wir
und
oder
ohne
mit
am
im
in
aus
auf
ist
sein
war
wird
ihr
ihre
ihres
als
für
von
mit
dich
dir
mich
mir
mein
sein
kein
durch
wegen
wird
a
an
and
are
as
at
be
but
by
for
if
in
into
is
it
no
not
of
on
or
s
such
t
that
the
their
then
there
these
they
this
to
was
will
with
EOD;
?>