Updated TripalFieldQuery to extend EntityFieldQuery so that the two are more interchangeable

tripal/api/tripal.fields.api.inc

@@ -1,5 +1,34 @@
 <?php
 
+/**
+ * Executes a TripalFieldQuery using the provided conditions.
+ *
+ * This hook is called to find the entities having certain field
+ * conditions and sort them in the given field order.
+ *
+ * @param $conditions
+ *   An array of filter representing the conditions to be applied to the query.
+ *   Each filter is an associative array whose keys include the following:
+ *   - field: an array representing the field identical to the output of the
+ *     field_info_field() function.
+ *   - filter: the name of the field on which the filter should be applied.
+ *   - value: the value of the filter.
+ *   - operator:  the operation to apply: '=', '<>', '>', '>=', '<', '<=',
+ *     'STARTS_WITH', 'CONTAINS': These operators expect $value to be a
+ *     literal of the same type as the column. 'IN', 'NOT IN': These operators
+ *     expect $value to be an array of literals of the same type as the column.
+ * @param $orderBy
+ *   An array of sorting instructions.  Each sort is an associative array with
+ *   the following keys:
+ *   - field: an array representing the field identical to the output of the
+ *     field_info_field() function.
+ *   - orderBy: the name of the field on which the filter should be applied.
+ *   - direction: either the string 'ASC' (for ascending sort) or 'DESC' (for
+ *     descending).
+ */
+function hook_field_storage_tquery($conditions, $orderBy) {
+  // See the tripal_chado_field_storage_tquery() function for an example.
+}
 /**
  * Retrieves a list of TripalField class instances for a given module.
  *

tripal/includes/TripalEntityUIController.inc

@@ -182,7 +182,7 @@ function tripal_view_entity($entity, $view_mode = 'full') {
      '#collapsed' => TRUE,
    );
    $etypes = db_select('tripal_bundle', 'tb')
-     ->fields('tb', array('id', 'label'))
+     ->fields('tb', array('name', 'label'))
      ->execute()
      ->fetchAllKeyed();
    $etypes = array('[any]' => 'any') +  $etypes;
@@ -206,11 +206,14 @@ function tripal_view_entity($entity, $view_mode = 'full') {
    );
 
    $query = new TripalFieldQuery();
+   $query->entityCondition('entity_type', 'TripalEntity');
    if ($filter_type) {
-     $query = $query->fieldCondition('content_type', $filter_type);
+     $query->entityCondition('bundle', $filter_type);
+     //$query->fieldCondition('content_type', 'content_type', 'organism');
+     //$query->fieldOrderBy('content_type', 'content_type', 'DESC');
+     //$query->fieldCondition('organism__genus', 'organism__genus', 'Oryza');
+
    }
-   //
-   //$query = $query->fieldCondition('feature__name', 'orange1', 'STARTS WITH');
    $results = $query->execute();
 
    $headers = array('Title', 'Vocabulary', 'Term', 'Author', 'Status', 'Updated', 'Operations');

tripal/includes/TripalFieldQuery.inc

@@ -2,156 +2,99 @@
 
 
 /**
- * A simple class for querying entities based on field values for fields.
- *
- * This class supports the use of multiple field storage. This class is loosely
- * modeled after the EntityFieldQuery class.
- *
+ * Extends the EntityFieldQuery to support queries from multiple storage types.
  */
-class TripalFieldQuery {
+class TripalFieldQuery extends EntityFieldQuery {
 
-  protected $options = array();
+  protected $field_storage = array();
 
-  /**
-   *
-   * @param $field_name
-   *   The name of the field.  Or, the name of the field with a subfield
-   *   concatenated using a '.' as a delimter.
-   * @param $value
-   *   The value to use for filtering.
-   * @param $operator
-   *   The operation to apply: '=', '<>', '>', '>=', '<', '<=', 'STARTS_WITH',
-   *   'CONTAINS': These operators expect $value to be a literal of the same
-   *   type as the column. 'IN', 'NOT IN': These operators expect $value to
-   *   be an array of literals of the same type as the column.
-   */
-  public function fieldCondition($field_name, $value, $operator = '=') {
+  public function execute() {
+    // Give a chance to other modules to alter the query.
+    drupal_alter('entity_query', $this);
+    $this->altered = TRUE;
 
-    // See if there is a subfield as part of the field_name.
-    $subfields = explode('.', $field_name);
-    if ($subfields > 1) {
-      $field = field_info_field($subfields[0]);
-    }
-    else {
-      $field = field_info_field($field_name);
-    }
+    // Initialize the pager.
+    $this->initializePager();
 
-    if ($field) {
-      $field_storage_type = $field['storage']['type'];
-      $this->options[$field_storage_type]['filters'][] = array(
-        'field' => $field,
-        'filter' => $field_name,
-        'value' => $value,
-        'operator' => $operator,
-      );
-    }
-    return $this;
-  }
+    // If there are fields then we need to support multiple backends, call
+    // the function for each one and merge the results.
+    if ($this->fields) {
 
-  /**
-   * Orders the result set by a given field column.
-   *
-   * @param $field_name
-   * @param $direction
-   *
-   * @return TripalFieldQuery
-   */
-  public function fieldOrderBy($field_name, $direction = 'ASC') {
+      // Build the list of all of the different field storage types used
+      // for this query.
+      foreach ($this->fields as $field) {
+        $this->field_storage[$field['storage']['type']] = $field['storage']['module'];
+      }
 
-    // See if there is a subfield as part of the field_name.
-    $subfields = explode('.', $field_name);
-    if ($subfields > 1) {
-      $field = field_info_field($subfields[0]);
+      // Initialize the results array.
+      $results = array();
+
+      // Iterate through the field storage types and call each one.
+      foreach ($this->field_storage as $storage_type => $storage_module) {
+        // Execute the query using the correct callback.
+        $callback = $this->queryStorageCallback($storage_module);
+        $st_results = call_user_func($callback, $this);
+        // If this is the first storage type to be queries then save these
+        // as the current results list.
+        if (count($results) == 0) {
+          $results = $st_results;
+        }
+        // If other results already exist then we want to find the intersection
+        // of the two and only save those.
+        else {
+          $intersection = array(
+            'TripalEntity' => array(),
+          );
+          foreach ($st_results['TripalEntity'] as $entity_id => $stub) {
+            if (array_key_exists($entity_id, $results['TripalEntity'])) {
+              $intersection['TripalEntity'][$entity_id] = $stub;
+            }
+          }
+          $results = $intersection;
+        }
+      }
     }
+    // If there are no fields then default to the original
+    // EntityFieldQuery() functionality.
     else {
-      $field = field_info_field($field_name);
-    }
-
-    if ($field) {
-      $field_storage_type = $field['storage']['type'];
-      $this->options[$field_storage_type]['sort'][] = array(
-        'field' => $field,
-        'orderBy' => $field_name,
-        'direction' => $direction,
-      );
+      $results = call_user_func($this->queryCallback(), $this);
     }
-    return $this;
+    return $results;
   }
 
+
   /**
-   * Executes the query and returns results.
+   * Determines the query callback to use for this entity query.
+   *
+   * This function is a replacement for queryCallback() from the
+   * parent EntityFieldQuery class because that class only allows a single
+   * storage type per query.
    *
-   * This function does not actually perform any queries itself but passes
-   * on the task to field storage backends for all of the fields in the
-   * filter.  Each backend should return a list of entity IDs that match
-   * the filters provided. The intersection of this list is returned.
+   * @param $storage
+   *   The storage module
    *
+   * @throws EntityFieldQueryException
    * @return
-   *  An array of associative arrays of stub entities. The result can be
-   *  used in the same way that results from the EntityFieldQuery->execute()
-   *  function are used.
+   *   A callback that can be used with call_user_func().
+   *
    */
-  public function execute() {
-    // Are there any filters?  If so, then let the field storage
-    // systems handle the query. If there are no fields then just pull out
-    // the list of entities.
-    // dpm($this->filters);
-    // dpm($this->sort);
-
-    $entity_ids = array();
-    if (count($this->options) > 0) {
-      // Iterate through each of the field storage types and run their
-      // tquery() function.
-      foreach ($this->options as $field_storage_type => $option) {
-        $filters = array_key_exists('filters', $option) ? $option['filters'] : array();
-        $sort = array_key_exists('sort', $option) ? $option['sort'] : array();
-
-        // Get the storage infor for the fields that belong to this type.
-        // We can get it from the first field.
-        if (count($filters) > 0) {
-          $storage = $filters[0]['field']['storage'];
-        }
-        else {
-          $storage = $sort[0]['field']['storage'];
-        }
-        $module = $storage['module'];
-        $function_name = $module . '_field_storage_tquery';
-        $filter_ids = array();
-        if (function_exists($function_name)) {
-          $filter_ids = $function_name($filters, $sort);
-        }
-        // Take the intersection of IDs in this filter with those in the
-        // final $entity_ids;
-        if (count($entity_ids) == 0) {
-          $entity_ids = $filter_ids;
-        }
-        else {
-          $entity_ids = array_intersect($entity_ids, $filter_ids);
-        }
-      }
+  public function queryStorageCallback($storage) {
+    // Use the override from $this->executeCallback. It can be set either
+    // while building the query, or using hook_entity_query_alter().
+    if (function_exists($this->executeCallback)) {
+      return $this->executeCallback;
     }
-    else {
-      $query = db_select('tripal_entity', 'td');
-      $query->fields('td', array('id'));
-      $query->orderBy('created', 'DESC');
-      $query->range(0,25);
-      $results = $query->execute();
-      while ($entity_id = $results->fetchField()) {
-        $entity_ids[] = $entity_id;
-      }
+    // If there are no field conditions and sorts, and no execute callback
+    // then we default to querying entity tables in SQL.
+    if (empty($this->fields)) {
+      return array($this, 'propertyQuery');
     }
-
-    // Order the entities by the field
-    if (count($this->order) > 0) {
-
+    if ($storage) {
+      // Use hook_field_storage_query() from the field storage.
+      return $storage . '_field_storage_query';
     }
-
-    // Generate the entities for the keys.
-    $return = array();
-    foreach ($entity_ids as $entity_id) {
-      $entity = entity_create_stub_entity('TripalEntity', array($entity_id, NULL, NULL));
-      $return['TripalEntity'][$entity_id] = $entity;
+    else {
+      throw new EntityFieldQueryException(t("Field storage engine not found."));
     }
-    return $return;
   }
 }

tripal/includes/tripal.field_storage.inc

@@ -62,61 +62,57 @@ function tripal_field_storage_load($entity_type, $entities, $age,
 
 /**
  * Implements hook_field_storage_query().
- *
- * Used by EntityFieldQuery to find the entities having certain field values.
- *
- * We do not support use of the EntityFieldQuery API for Tripal based fields
- * because EFQ doesn't support when multiple storage backends are used. Instead
- * use the TripalFieldQuery class and implement the hook_storage_tquery()
- * function.
  */
 function tripal_field_storage_query($query) {
 
-}
-
-/**
- * Implements hook_field_storage_tquery().
- *
- * Used by TripalFieldQuery to find the entities having certain field values.
- *
- * @param $conditions
- */
-function tripal_field_storage_tquery($conditions, $sort) {
-
   $filter = array();
   $entity_ids = array();
 
-  foreach ($conditions as $index => $condition) {
+  // Create the initial query.
+  $select = db_select('tripal_entity', 'TE');
+  $select->join('tripal_bundle', 'TB', 'TE.bundle = TB.name');
+  $select->fields('TE', array('id'));
+  $select->fields('TB', array('name'));
 
+  // Add in any filters to the query.
+  foreach ($query->fieldConditions as $index => $condition) {
     $field = $condition['field'];
+    // Skip conditions that don't belong to this storage type.
     if ($field['storage']['type'] != 'tripal_no_storage') {
       continue;
     }
-    $field_type = $field['type'];
-    $field_module = $field['module'];
-    $settings = $field['settings'];
-    $operator = $condition['operator'];
-
-    // This module only supports one field, so we can just perform the filter.
-    // using the appropriate operator.
-    $query = db_select('tripal_entity', 'TE');
-    $query->join('chado_entity', 'CE', 'TE.id = CE.entity_id');
-    $query->fields('CE', array('entity_id'));
-    $query->condition('TE.term_id', $condition['value'], $operator);
-    $results = $query->execute();
-    $filter_ids = array();
-    while ($entity_id = $results->fetchField()) {
-      $filter_ids[] = $entity_id;
+    $value = $condition['value'];
+    $operator = $condition['operator'] ? $condition['operator'] : '=';
+
+    // Filtering on the content type is filtering on the label.
+    if ($field['field_name'] == 'content_type') {
+      $select->condition('TB.label', $value, $operator);
     }
+  }
 
-    // Take the intersection of IDs in this filter with those in the
-    // final $entity_ids;
-    if (count($entity_ids) == 0) {
-      $entity_ids = $filter_ids;
+  // Add in any sorting to the query.
+  foreach ($query->order as $index => $sort) {
+    $field = $sort['specifier']['field'];
+    // Skip sorts that don't belong to this storage type.
+    if ($field['storage']['type'] != 'tripal_no_storage') {
+      continue;
     }
-    else {
-      $entity_ids = array_intersect($entity_ids, $filter_ids);
+    $direction = $sort['direction'];
+
+    // Filtering on the content type is a filter using the label
+    if ($field['field_name'] == 'content_type') {
+      $select->orderBy('TB.label', $direction);
     }
   }
-  return $entity_ids;
+
+  // Perform the query and return the results.
+  $entities = $select->execute();
+  $result = array(
+    'TripalEntity' => array(),
+  );
+  while ($entity = $entities->fetchObject()) {
+    $ids = array($entity->id, '0', $entity->name);
+    $result['TripalEntity'][$entity->id] = entity_create_stub_entity('TripalEntity', $ids);
+  }
+  return $result;
 }

tripal/tripal.install

@@ -451,6 +451,7 @@ function tripal_tripal_entity_schema() {
       'entity_created' => array('created'),
       'type' => array('type'),
       'uid' => array('uid'),
+      'bundle' => array('bundle'),
     ),
     'unique keys' => array(),
     'primary key' => array('id'),
@@ -627,6 +628,7 @@ function tripal_tripal_bundle_schema() {
     'indexes' => array(
       'name' => array('name'),
       'term_id' => array('term_id'),
+      'label' => array('label'),
     ),
     'primary key' => array('id'),
     'unique keys' => array(

tripal/tripal_views_query.inc

@@ -35,6 +35,7 @@ class tripal_views_query extends views_plugin_query {
     // must be with the content_type field.
     $query = new TripalFieldQuery();
     $query->fieldCondition('content_type', $bundle->id);
+
     // Apply filters
     foreach ($view->filter as $field_name => $handler) {
       if (trim($handler->value)) {
@@ -53,11 +54,11 @@ class tripal_views_query extends views_plugin_query {
       foreach ($results['TripalEntity'] as $entity_id => $stub) {
         // Begin a new row for Views output.
         $row = new stdClass;
-  
+
         // Get the entity object.
         $entity = tripal_load_entity('TripalEntity', array('id' => $entity_id));
         $entity = reset($entity);
-  
+
         // Iterate through the fields that are added to the view and attach those
         // to the entity.  After attaching we can get the value and include
         // it in the output results.
@@ -67,14 +68,14 @@ class tripal_views_query extends views_plugin_query {
               array('field_id' => $field['id']));
           $items = field_get_items('TripalEntity', $entity, $field_name);
           $value = $items[0]['value'];
-  
+
           $row->entity = $entity;
           $row->$field_name = $value;
         }
-  
+
         // Add the row to the results list.
         $view->result[] = $row;
-  
+
       }
     }
 

tripal_chado/includes/tripal_chado.field_storage.inc

@@ -405,24 +405,29 @@ function tripal_chado_field_storage_expand_field($item_name, $value) {
      return array($item_name => $value);
    }
 }
+
 /**
- * Implements hook_field_storage_tquery().
- *
- * Used by TripalFieldQuery to find the entities having certain field values.
- *
- * @param $conditions
+ * Implements hook_field_storage_query().
  */
-function tripal_chado_field_storage_tquery($conditions, $sort) {
+function tripal_chado_field_storage_query($query) {
+dpm($query);
+  // The conditions and order bys are reorganized into a filters array for the
+  // chado_select_record function()
   $filters = array();
-  $entity_ids = array();
 
-  foreach ($conditions as $index => $condition) {
+  // Iterate through all the conditions and add to the filters array
+  // a chado_select_record compatible set of filters.
+  foreach ($query->fieldConditions as $index => $condition) {
     $field = $condition['field'];
-    $filter = $condition['filter'];
 
+    // Skip conditions that don't belong to this storage type.
     if ($field['storage']['type'] != 'field_chado_storage') {
       continue;
     }
+
+    $column = $condition['column'];
+    $value = $condition['value'];
+
     $field_type = $field['type'];
     $field_module = $field['module'];
     $settings = $field['settings'];
@@ -431,7 +436,7 @@ function tripal_chado_field_storage_tquery($conditions, $sort) {
 
     // Set the value for this field search.
     $value = NULL;
-    $subfields = explode('.', $filter);
+    $subfields = explode('.', $column);
     //print_r($subfields);
     if (count($subfields) > 1) {
       // Get the term for this field's column and replace the field_name with
@@ -441,7 +446,7 @@ function tripal_chado_field_storage_tquery($conditions, $sort) {
       // web services.
       $subfield1 = tripal_get_chado_semweb_term($chado_table, $chado_column, array('return_object' => TRUE));
       $subfields[0] = strtolower(preg_replace('/ /', '_', $subfield1->name));
-      $value = tripal_chado_field_storage_recurse_subfilters($chado_table, $subfields, $condition['value']);
+      $value = tripal_chado_field_storage_recurse_subfilters($chado_table, $subfields, $value);
       $value = array_shift($value);
     }
     else {
@@ -449,7 +454,7 @@ function tripal_chado_field_storage_tquery($conditions, $sort) {
     }
 
     // Use the appropriate operator.
-    $operator = $condition['operator'];
+    $operator = $condition['operator'] ? $condition['operator'] : '=';
     switch ($operator) {
       case '=':
         $filters[$chado_table][$chado_column] = $value;
@@ -511,6 +516,26 @@ function tripal_chado_field_storage_tquery($conditions, $sort) {
     }
   }
 
+  // Now get the list for sorting.
+  foreach ($query->order as $index => $sort) {
+    $field = $sort['specifier']['field'];
+
+    // Skip sorts that don't belong to this storage type.
+    if ($field['storage']['type'] != 'field_chado_storage') {
+      continue;
+    }
+
+    $direction = $sort['direction'];
+
+    $field_type = $field['type'];
+    $field_module = $field['module'];
+    $settings = $field['settings'];
+    $chado_table = $settings['chado_table'];
+    $chado_column = $settings['chado_column'];
+
+    $sorting[$chado_table][$chado_column] = $direction;
+  }
+
   // Iterate through the filters and perform the query
   $entity_ids = array();
   foreach ($filters as $chado_table => $values) {
@@ -529,26 +554,25 @@ function tripal_chado_field_storage_tquery($conditions, $sort) {
     // Next look for matching IDs in the chado_entity table.
     $filter_ids = array();
     if (count($record_ids) > 0) {
-      $results = db_select('chado_entity', 'CE')
-        ->fields('CE', array('entity_id'))
-        ->condition('record_id', $record_ids)
-        ->execute();
+      $select = db_select('chado_entity', 'CE');
+      $select->join('tripal_entity', 'TE', 'TE.id = CE.entity_id');
+      $select->fields('CE', array('entity_id'));
+      $select->fields('TE', array('bundle'));
+      $select->condition('record_id', $record_ids);
+      $results = $select->execute();
       while ($result = $results->fetchObject()) {
-        $filter_ids[] = $result->entity_id;
+        $entity_ids[] = array($result->entity_id, 0, $result->bundle);
       }
     }
-
-    // Take the intersection of IDs in this filter with those in the
-    // final $entity_ids;
-    if (count($entity_ids) == 0) {
-      $entity_ids = $filter_ids;
-    }
-    else {
-      $entity_ids = array_intersect($entity_ids, $filter_ids);
-    }
   }
 
-  return $entity_ids;
+  $result = array(
+    'TripalEntity' => array()
+  );
+  foreach ($entity_ids as $ids) {
+    $result['TripalEntity'][$ids[0]] = entity_create_stub_entity('TripalEntity', $ids);
+  }
+  return $result;
 }
 
 /**

tripal_chado/includes/tripal_chado.install.inc

@@ -20,12 +20,18 @@ function tripal_chado_install_form() {
 
   if ($real_version == '1.2') {
     drupal_set_message('Please note: the upgrade of Chado from v1.2 to v1.3 may
-        require a fix to your materialized views.  All of the primary keys
+        require three fixes to your database. All of the primary keys
         in Chado were changed from integers to big integers to support larger
-        tables.  If your materialized views uses these fields you may need to
-        alter and repopulate those views.  Additionally, if you have made
+        tables.  First, if your site has custom materialized views that will hold
+        data derived from fields changed to big integers then you may need to
+        alter the views to change the fields from integers to big integers
+        and repopulate those views.  Second, if you have made
         any custom PL/pgSQL functions that expect primary and foreign key fields
-        to be integers, then those functions will need to be correct.
+        to be integers, then those functions will need to be altered to accept
+        big integers.  Third, if you have PostgreSQL Views that use fields
+        that are converted to big integers then most likely this upgrade will
+        fail.  You must first remove those views, perform the upgrade and then
+        recreate them with the appropriate fields change to big integers.
         The Tripal upgrader is not able to fix these problems automatically',
         'warning');
   }