fixing merge conflict

tripal/api/tripal.collections.api.inc

@@ -25,6 +25,10 @@
 /**
  * Creates a collection of entities for a given user.
  *
+ * Use this function if you want to create a new collection with an bundle.
+ * Otherwise, a new colleciton can also be created by creating an new instance
+ * of a TripalEntityCollection object.
+ *
  * @param  $details
  *   An association array containing the details for a collection. The
  *   details must include the following key/value pairs:
@@ -41,33 +45,29 @@
  * @ingroup tripal_data_collections_api
  */
 function tripal_create_collection($details) {
+
   global $user;
 
   try {
     $collection = new TripalEntityCollection();
     $collection->create($details);
     $collection_id = $collection->getCollectionID();
+    $collection->addBundle($details);
 
-    // Add the job to write the collection download files.
-    $args = array($collection_id);
-    tripal_add_job('Create data collection files for ' . $user->name, 'tripal',
-        'tripal_create_collection_files', $args, $user->uid, 10, array());
-
-
-    drupal_set_message(t("Collection '%name' created with %num_recs record(s).  Downloadble files will be available shortly.  Check the !view for status and download links.",
+    drupal_set_message(t("Collection '%name' created with %num_recs record(s). Check the !view to generate file links.",
       array(
         '%name' => $details['collection_name'],
         '%num_recs' => count($details['ids']),
         '!view' => l('data collections page', 'user/' . $user->uid . '/data-collections'),
       ))
     );
-
+    return $collection;
   }
   catch (Exception $e) {
     drupal_set_message(t("Failed to create the collection '%name': " . $e->getMessage(), array('%name' =>  $details['collection_name'])), 'error');
     return FALSE;
   }
-  return $collection;
+
 }
 
 /**
@@ -109,12 +109,12 @@ function tripal_get_user_collections($uid) {
  * @ingroup tripal_data_collections_api
  */
 function tripal_expire_collections() {
-  $max_hours = variable_get('tripal_data_collections_lifespan', 7);
+  $max_days = variable_get('tripal_data_collections_lifespan', 30);
   $ctime = time();
 
   $query = db_select('tripal_collection', 'tc');
   $query->fields('tc', array('collection_id'));
-  $query->where("(($ctime - create_date) / 60) / 60 >= $max_hours");
+  $query->where("((($ctime - create_date) / 60) / 60) / 24 >= $max_days");
   $results = $query->execute();
   while ($collection_id = $results->fetchField()) {
     $collection = new TripalEntityCollection();
@@ -188,15 +188,7 @@ function tripal_get_collection($values) {
  * @ingroup tripal_data_collections_api
  */
 function tripal_create_collection_files($collection_id, TripalJob $job = NULL) {
-   if($job) {
-     $job->setProgress(0);
-   }
-
-   $collection = new TripalEntityCollection();
-   $collection->load($collection_id);
-   $collection->writeAll();
-
-   if ($job) {
-     $job->setProgress(100);
-   }
-}
+  $collection = new TripalEntityCollection();
+  $collection->load($collection_id);
+  $collection->write(NULL, $job);
+}

tripal/api/tripal.entities.api.inc

@@ -158,7 +158,11 @@ function hook_tripal_default_title_format($bundle, $available_tokens) {
  * @param $field_ids
  *   A list of numeric field IDs that should be loaded.  The
  *   TripalField named 'content_type' is always automatically added.
- *
+ * @param $cache
+ *  When loading of entities they can be cached with Drupal for later
+ *  faster loading. However, this can cause memory issues when running
+ *  Tripal jobs that load lots of entities.  Caching of entities can
+ *  be disabled to improve memory performance by setting this to FALSE.
  * @return
  *   An array of entity objects indexed by their ids. When no results are
  *   found, an empty array is returned.
@@ -166,7 +170,7 @@ function hook_tripal_default_title_format($bundle, $available_tokens) {
  * @ingroup tripal_entities_api
  */
 function tripal_load_entity($entity_type, $ids = FALSE, $reset = FALSE,
-    $field_ids = array()) {
+    $field_ids = array(), $cache = TRUE) {
 
   // The $conditions is deprecated in the funtion arguments of entity_load
   // so it was removed from the parameters of this function as well. But
@@ -185,7 +189,8 @@ function tripal_load_entity($entity_type, $ids = FALSE, $reset = FALSE,
   if ($reset) {
     $ec->resetCache();
   }
-  return $ec->load($ids, $conditions, $field_ids);
+
+  return $ec->load($ids, $conditions, $field_ids, $cache);
 }
 /**
  * Retrieves a TripalTerm entity that matches the given arguments.
@@ -1190,8 +1195,15 @@ function tripal_entity_label($entity) {
 }
 
 /**
+ * Retrieves details, including attached fields, for a given bundle.
  *
  * @param $bundle_name
+ *   The name of the bundle (e.g. bio_data_xx)
+ *
+ * @return
+ *   An array containing the name, label, controlled vocabulary details
+ *   and a list of fields attached to the bundle.  Returns FALSE
+ *   if the bundle does not exist.
  *
  * @ingroup tripal_entities_api
  */
@@ -1199,6 +1211,9 @@ function tripal_get_bundle_details($bundle_name) {
   global $user;
 
   $bundle = tripal_load_bundle_entity(array('name' => $bundle_name));
+  if (!$bundle) {
+    return FALSE;
+  }
   $term = tripal_load_term_entity(array('term_id' => $bundle->term_id));
   $vocab = $term->vocab;
   $instances = field_info_instances('TripalEntity', $bundle->name);

tripal/api/tripal.fields.api.inc

@@ -220,32 +220,34 @@ function tripal_get_field_widgets() {
  *
  * @param $field
  *   A field array as returned by the field_info_field() function.
+ * @param $instance
+ *   An field instance array.
  * @return
  *   A list of file formatter class names.
  */
-function tripal_get_field_field_formatters($field) {
+function tripal_get_field_field_formatters($field, $instance) {
   $field_name = $field['field_name'];
   $field_type = $field['type'];
 
   $downloaders = array();
 
-  // All fields should support the Tab and CSV downloaders.
-  tripal_load_include_downloader_class('TripalTabDownloader');
-  $downloaders['TripalTabDownloader'] = TripalTabDownloader::$label;
-  tripal_load_include_downloader_class('TripalCSVDownloader');
-  $downloaders['TripalCSVDownloader'] = TripalCSVDownloader::$label;
-
+  // If the field type is a TripalField then get information about the formatter.
   if (tripal_load_include_field_class($field_type)) {
-    $settings = $field_type::$default_instance_settings;
-    if (array_key_exists('download_formatters', $settings)) {
-      foreach ($settings['download_formatters'] as $class_name) {
-        if (!array_key_exists($class_name, $downloaders)) {
-          tripal_load_include_downloader_class($class_name);
-          $downloaders[$class_name] = $class_name::$label;
-        }
+    $formatters = $field_type::$download_formatters;
+    foreach ($formatters as $class_name) {
+      if (!array_key_exists($class_name, $downloaders)) {
+        tripal_load_include_downloader_class($class_name);
+        $downloaders[$class_name] = $class_name::$full_label;
       }
     }
   }
+  else {
+    // For non TripalFields we'll assume TAB and CSV.
+    tripal_load_include_downloader_class('TripalTabDownloader');
+    tripal_load_include_downloader_class('TripalCSVDownloader');
+    $downloaders['TripalTabDownloader'] = TripalTabDownloader::$full_label;
+    $downloaders['TripalCSVDownloader'] = TripalCSVDownloader::$full_label;
+  }
   return $downloaders;
 }
 

tripal/api/tripal.terms.api.inc

@@ -346,7 +346,7 @@ function tripal_add_term($details) {
 function tripal_get_term_details($vocabulary, $accession) {
 
   if (empty($vocabulary) OR empty($accession)) {
-    tripal_report_error('tripal_term', TRIPAL_ERROR, 'Unable to retrieve details for term due to missing vocabulary and/or accession.');
+    tripal_report_error('tripal_term', TRIPAL_ERROR, "Unable to retrieve details for term due to missing vocabulary and/or accession");
   }
 
   // TODO: we need some sort of administrative interface that lets the user

tripal/includes/TripalBundle.inc

@@ -7,6 +7,8 @@ class TripalBundle extends Entity {
   public function __construct($values = array(), $entity_type) {
     parent::__construct($values, $entity_type);
 
+    $this->term = tripal_load_term_entity(array('term_id' => $this->term_id));
+    $vocab = $this->term->vocab;
+    $this->accession = $vocab->vocabulary . ':' . $this->term->accession;
   }
-
 }

tripal/includes/TripalEntityCollection.inc

@@ -3,9 +3,9 @@
 class TripalEntityCollection {
 
   /**
-   * The name of the bundle (i.e. content type) to which the entities belong.
+   * The name of the bundles (i.e. content type) to which the entities belong.
    */
-  protected $bundle_name = '';
+  protected $bundles = array();
 
   /**
    * The collection ID
@@ -36,7 +36,7 @@ class TripalEntityCollection {
   /**
    * The list of downloaders available for this bundle.
    */
-  protected $downloaders = array();
+  protected $formatters = array();
 
   /**
    * The description for this collection.
@@ -60,31 +60,33 @@ class TripalEntityCollection {
     }
 
     try {
-      db_delete('tripal_collection')
-        ->condition('collection_id', $this->collection_id)
-        ->execute();
-
       // Remove any files that may have been created
-      foreach ($this->downloaders as $class_name => $label) {
+      foreach ($this->formatters as $class_name => $label) {
         tripal_load_include_downloader_class($class_name);
         $outfile = $this->getOutfile($class_name);
-        $downloader = new $class_name($this->bundle_name, $this->ids, $this->fields,
-            $outfile, $this->getUserID());
+        $downloader = new $class_name($this->collection_id, $outfile);
         $downloader->delete();
       }
 
+      // Delete from the tripal collection table.
+      db_delete('tripal_collection')
+        ->condition('collection_id', $this->collection_id)
+        ->execute();
+
+      // Delete the field groups from the tripal_bundle_collection table.
+      db_delete('tripal_collection_bundle')
+        ->condition('collection_id', $this->collection_id)
+        ->execute();
+
       // Reset the class to defaults.
       $this->collection_id = NULL;
-      $this->bundle_name = '';
       $this->collection_name = '';
       $this->create_date = '';
       $this->description = '';
-      $this->fields = array();
-      $this->ids = array();
 
     }
     catch (Exception $e) {
-      throw new Exception('Cannot delete collection: ' .  $e->getMessage());
+      throw new Exception('Cannot delete collection: ' . $e->getMessage());
     }
   }
 
@@ -114,59 +116,42 @@ class TripalEntityCollection {
     }
 
     // Fix the date/time fields.
-    $this->bundle_name = $collection->bundle_name;
     $this->collection_name = $collection->collection_name;
     $this->create_date = $collection->create_date;
     $this->user = user_load($collection->uid);
-    $this->ids = unserialize($collection->ids);
-    $this->fields = unserialize($collection->fields);
     $this->description = $collection->description;
     $this->collection_id = $collection->collection_id;
 
+    // Now get the bundles in this collection.
+    $bundles = db_select('tripal_collection_bundle', 'tcb')
+      ->fields('tcb')
+      ->condition('collection_id', $collection->collection_id)
+      ->execute();
+
+    // If more than one bundle plop into associative array.
+    while ($bundle = $bundles->fetchObject()) {
+      $bundle_name = $bundle->bundle_name;
+      $this->bundles[$bundle_name] = $bundle;
+      $this->ids[$bundle_name] = unserialize($bundle->ids);
+      $this->fields[$bundle_name] = unserialize($bundle->fields);
+    }
+
     // Iterate through the fields and find out what download formats are
     // supported for this basket.
-    foreach ($this->fields as $field_id) {
-      $field = field_info_field_by_id($field_id);
-      if (!$field) {
-        continue;
-      }
-      $field_name = $field['field_name'];
-      $field_type = $field['type'];
-      $field_module = $field['module'];
-      $instance = field_info_instance('TripalEntity', $field_name, $this->bundle_name);
-      $downloaders = array();
-
-      // All fields should support the Tab and CSV downloaders.
-      tripal_load_include_downloader_class('TripalTabDownloader');
-      $this->downloaders['TripalTabDownloader'] = TripalTabDownloader::$label;
-      tripal_load_include_downloader_class('TripalCSVDownloader');
-      $this->downloaders['TripalCSVDownloader'] = TripalCSVDownloader::$label;
-
-      if (tripal_load_include_field_class($field_type)) {
-        $settings = $field_type::$default_instance_settings;
-        if (array_key_exists('download_formatters', $settings)) {
-          foreach ($settings['download_formatters'] as $class_name) {
-            if (!array_key_exists($class_name, $this->downloaders)) {
-              tripal_load_include_downloader_class($class_name);
-              $this->downloaders[$class_name] = $class_name::$label;
-            }
-          }
-        }
-      }
-    }
+    $this->formatters = $this->setFormatters();
   }
 
   /**
-   * Creates a new collection.
+   * Creates a new data collection.
+   *
+   * To add bundles with entities and fields to a collection, use the
+   * addBundle() function after the collection is created.
    *
    * @param  $details
    *   An association array containing the details for a collection. The
    *   details must include the following key/value pairs:
    *   - uid:  The ID of the user that owns the collection
    *   - collection_name:  The name of the collection
-   *   - bundle_name:  The name of the TripalEntity content type.
-   *   - ids:  An array of the entity IDs that form the collection.
-   *   - fields: An array of the field IDs that the collection is limited to.
    *   - description:  A user supplied description for the collection.
    *
    * @throws Exception
@@ -178,24 +163,9 @@ class TripalEntityCollection {
     if (!$details['collection_name']) {
       throw new Exception("Must provide a 'collection_name' key to TripalEntityCollection::create().");
     }
-    if (!$details['bundle_name']) {
-      throw new Exception("Must provide a 'bundle_name' key to TripalEntityCollection::create().");
-    }
-    if (!$details['ids']) {
-      throw new Exception("Must provide a 'ids' key to TripalEntityCollection::create().");
-    }
-    if (!$details['fields']) {
-      throw new Exception("Must provide a 'fields' key to TripalEntityCollection::create().");
-    }
 
-    if (!is_array($details['ids'])) {
-      throw new Exception("The 'ids' key must be an array key to TripalEntityCollection::create().");
-    }
-    if (!is_array($details['fields'])) {
-      throw new Exception("The 'ids' key must be an array key to TripalEntityCollection::create().");
-    }
 
-    // Before inserting the new collection make sure we don't violote the unique
+    // Before inserting the new collection make sure we don't violate the unique
     // constraint that a user can only have one collection of the give name.
     $has_match = db_select('tripal_collection', 'tc')
       ->fields('tc', array('collection_id'))
@@ -211,20 +181,118 @@ class TripalEntityCollection {
       $collection_id = db_insert('tripal_collection')
         ->fields(array(
           'collection_name' => $details['collection_name'],
-          'bundle_name' => $details['bundle_name'],
-          'ids' => serialize($details['ids']),
-          'fields' => serialize($details['fields']),
           'create_date' => time(),
           'uid' => $details['uid'],
           'description' => array_key_exists('description', $details) ? $details['description'] : '',
         ))
         ->execute();
+
       // Now load the job into this object.
       $this->load($collection_id);
     }
     catch (Exception $e) {
-      throw new Exception('Cannot create collection: ' .  $e->getMessage());
+      throw new Exception('Cannot create collection: ' . $e->getMessage());
+    }
+  }
+
+  /**
+   * Creates a new tripal_collection_bundle entry.
+   *
+   * @param  $details
+   *   An association array containing the details for a collection. The
+   *   details must include the following key/value pairs:
+   *   - bundle_name:  The name of the TripalEntity content type.
+   *   - ids:  An array of the entity IDs that form the collection.
+   *   - fields: An array of the field IDs that the collection is limited to.
+   *
+   * @throws Exception
+   */
+  public function addBundle($details) {
+    if (!$details['bundle_name']) {
+      throw new Exception("Must provide a 'bundle_name' to TripalEntityCollection::addFields().");
+    }
+    if (!$details['ids']) {
+      throw new Exception("Must provide a 'ids' to TripalEntityCollection::addFields().");
+    }
+    if (!$details['fields']) {
+      throw new Exception("Must provide a 'fields' to TripalEntityCollection::addFields().");
+    }
+
+    try {
+      $collection_bundle_id = db_insert('tripal_collection_bundle')
+        ->fields(array(
+          'bundle_name' => $details['bundle_name'],
+          'ids' => serialize($details['ids']),
+          'fields' => serialize($details['fields']),
+          'collection_id' => $this->collection_id,
+        ))
+        ->execute();
+
+      // Now load the job into this object.
+      $this->load($this->collection_id);
+    }
+    catch (Exception $e) {
+      throw new Exception('Cannot create collection: ' . $e->getMessage());
+    }
+  }
+
+  /**
+   * Retrieves the list of bundles associated with the collection.
+   *
+   * @return
+   *   An array of bundles.
+   */
+  public function getBundles() {
+    return $this->bundles;
+  }
+
+  /**
+   * Retrieves the site id for this specific bundle fo the collection.
+   *
+   * @return
+   *   A remote site ID, or an empty string if the bundle is local.
+   */
+  public function getBundleSiteId($bundle_name) {
+    return $this->bundles[$bundle_name]->site_id;
+  }
+
+  /**
+   * Retrieves the list of appropriate download formatters for the basket.
+   *
+   * @return
+   *   An associative array where the key is the TripalFieldDownloader class
+   *   name and the value is the human-readable lable for the formatter.
+   */
+  private function setFormatters() {
+
+    $downloaders = array();
+    // Iterate through the fields and find out what download formats are
+    // supported for this basket.
+    foreach ($this->fields as $bundle_name => $field_ids) {
+
+      // Need the site ID from the tripal_collection_bundle table.
+      $site_id = $this->getBundleSiteId($bundle_name);
+
+      foreach ($field_ids as $field_id) {
+        // If this is a field from a remote site then get it's formatters
+        if ($site_id and module_exists('tripal_ws')) {
+          $formatters = tripal_get_remote_field_formatters($site_id, $bundle_name, $field_id);
+          $this->formatters += $formatters;
+        }
+        else {
+          $field_info = field_info_field_by_id($field_id);
+          if (!$field_info) {
+            continue;
+          }
+          $field_name = $field_info['field_name'];
+          $instance = field_info_instance('TripalEntity', $field_name, $bundle_name);
+          $formatters = tripal_get_field_field_formatters($field_info, $instance);
+          $this->formatters += $formatters;
+        }
+      }
     }
+    $this->formatters = array_unique($this->formatters);
+    return $this->formatters;
   }
 
   /**
@@ -234,8 +302,8 @@ class TripalEntityCollection {
    *   An associative array where the key is the TripalFieldDownloader class
    *   name and the value is the human-readable lable for the formatter.
    */
-  public function getDownloadFormatters() {
-     return $this->downloaders;
+  public function getFormatters() {
+    return $this->formatters;
   }
 
   /**
@@ -244,8 +312,8 @@ class TripalEntityCollection {
    * @return
    *   An array of numeric entity IDs.
    */
-  public function getEntityIDs(){
-    return $this->ids;
+  public function getEntityIDs($bundle_name) {
+    return $this->ids[$bundle_name];
   }
 
   /**
@@ -254,8 +322,8 @@ class TripalEntityCollection {
    * @return
    *   An array of numeric field IDs.
    */
-  public function getFields() {
-    return $this->fields;
+  public function getFieldIDs($bundle_name) {
+    return $this->fields[$bundle_name];
   }
 
   /**
@@ -348,7 +416,6 @@ class TripalEntityCollection {
     $extension = $formatter::$default_extension;
     $create_date = $this->getCreateDate(FALSE);
     $outfile = preg_replace('/[^\w]/', '_', ucwords($this->collection_name)) . '_collection' . '_' . $create_date . '.' . $extension;
-
     return $outfile;
   }
 
@@ -360,8 +427,8 @@ class TripalEntityCollection {
    * @return boolean
    *   TRUE if the formatter is compatible, FALSE otherwise.
    */
-  public function isFormatterCompatible($formatter) {
-    foreach ($this->downloaders as $class_name => $label) {
+  protected function isFormatterCompatible($formatter) {
+    foreach ($this->formatters as $class_name => $label) {
       if ($class_name == $formatter) {
         return TRUE;
       }
@@ -369,17 +436,6 @@ class TripalEntityCollection {
     return FALSE;
   }
 
-  /**
-   * Writes the collection to all file downloadable formats.
-   *
-   * @throws Exception
-   */
-  public function writeAll() {
-    foreach ($this->downloaders as $class_name => $lable) {
-      $this->write($class_name);
-    }
-  }
-
   /**
    * Retrieves the URL for the downloadable file.
    *
@@ -388,7 +444,6 @@ class TripalEntityCollection {
    */
   public function getOutfileURL($formatter) {
     $outfile = $this->getOutfilePath($formatter);
-    return file_create_url($outfile);
   }
 
   /**
@@ -400,73 +455,124 @@ class TripalEntityCollection {
    *   The name of the class
    */
   public function getOutfilePath($formatter) {
-    if(!$this->isFormatterCompatible($formatter)) {
+    if (!$this->isFormatterCompatible($formatter)) {
       throw new Exception(t('The formatter, "@formatter", is not compatible with this data collection.', array('@formatter' => $formatter)));
-
     }
-
     if (!tripal_load_include_downloader_class($formatter)) {
       throw new Exception(t('Cannot find the formatter named "@formatter".', array('@formatter', $formatter)));
     }
 
     $outfile = $this->getOutfile($formatter);
-
-    $downloader = new $formatter($this->bundle_name, $this->ids, $this->fields, $outfile, $this->user->uid);
-
-    return $downloader->getURL();
+    // Make sure the user directory exists
+    $user_dir = 'public://tripal/users/' . $this->user->uid;
+    $outfilePath = $user_dir. '/' . $outfile;
+    return $outfilePath;
   }
 
   /**
-   * Writes the collection to a file.
+   * Writes the collection to a file using a given formatter.
    *
    * @param formatter
    *   The name of the formatter class to use (e.g. TripalTabDownloader). The
-   *   formatter must be compatible with the data collection.
-   *
+   *   formatter must be compatible with the data collection.  If no
+   *   formatter is supplied then all file formats supported by this
+   *   data collection will be created.
+   * @param $job
+   *    If this function is run as a Tripal Job then this argument can be
+   *    set to the Tripaljob object for keeping track of progress.
    * @throws Exception
    */
-  public function write($formatter) {
-
-    if(!$this->isFormatterCompatible($formatter)) {
-      throw new Exception(t('The formatter, "@formatter", is not compatible with this data collection.', array('@formatter' => $formatter)));
+  public function write($formatter = NULL, TripalJob $job = NULL) {
 
+    // Initialize the downloader classes and initialize the files for writing.
+    $formatters = array();
+    foreach ($this->formatters as $class => $label) {
+      if (!$this->isFormatterCompatible($class)) {
+        throw new Exception(t('The formatter, "@formatter", is not compatible with this data collection.', array('@formatter' => $formatter)));
+      }
+      if (!tripal_load_include_downloader_class($class)) {
+        throw new Exception(t('Cannot find the formatter named "@formatter".', array('@formatter', $formatter)));
+      }
+      $outfile = $this->getOutfile($class);
+      if (!$formatter or ($formatter == $class)) {
+        $formatters[$class] = new $class($this->collection_id, $outfile);
+        $formatters[$class]->writeInit($job);
+        if ($job) {
+          $job->logMessage("Writing " . lcfirst($class::$full_label) . " file.");
+        }
+      }
     }
 
-    if (!tripal_load_include_downloader_class($formatter)) {
-      throw new Exception(t('Cannot find the formatter named "@formatter".', array('@formatter', $formatter)));
+    // Count the total number of entities
+    $total_entities = 0;
+    $bundle_collections = $this->collection_bundles;
+    foreach ($this->bundles as $bundle) {
+      $bundle_name = $bundle->bundle_name;
+      $entity_ids  = $this->getEntityIDs($bundle_name);
+      $total_entities += count($entity_ids);
+    }
+    if ($job) {
+      $job->setTotalItems($total_entities);
     }
 
-    $outfile = $this->getOutfile($formatter);
+    // Iterate through the bundles in this collection and get the entities.
+    foreach ($this->bundles as $bundle) {
+      $bundle_name = $bundle->bundle_name;
+      $site_id = $bundle->site_id;
+      $entity_ids  = array_unique($this->getEntityIDs($bundle_name));
+      $field_ids = array_unique($this->getFieldIDs($bundle_name));
 
-    // Filter out fields that aren't supported by the formatter.
-    $supported_fields = array();
-    foreach ($this->fields as $field_id) {
-      // If the formatter is TripalTabDownloader or TripalCSVDownloader then
-      // we always want to support the field.
-      if ($formatter == 'TripalTabDownloader' or $formatter == 'TripalCSVDownloader') {
-        if (!in_array($field_id, $supported_fields)) {
-          $supported_fields[] = $field_id;
-        }
-        continue;
+      // Clear any cached @context or API docs.
+      if ($site_id and module_exists('tripal_ws')) {
+        tripal_clear_remote_cache($site_id);
       }
 
-      // Otherwise, find out if the formatter specified is supporte by the
-      // field and if so then add it to our list of supported fields.
-      $field = field_info_field_by_id($field_id);
-      $field_name = $field['field_name'];
-      $field_type = $field['type'];
-      if (tripal_load_include_field_class($field_type)) {
-        $settings = $field_type::$default_instance_settings;
-        if (array_key_exists('download_formatters', $settings)) {
-          if (in_array($formatter, $settings['download_formatters'])) {
-            $supported_fields[] = $field_id;
+      // We want to load entities in batches to speed up performance.
+      $num_eids = count($entity_ids);
+      $bundle_eids_handled = 0;
+      $slice_size = 100;
+      while ($bundle_eids_handled < $num_eids) {
+        // Get a bantch of $slice_size elements from the entities array.
+        $slice = array_slice($entity_ids, $bundle_eids_handled, $slice_size);
+        if ($job) {
+          $job->logMessage('Getting entities for ids !start to !end of !total',
+              array('!start' => $bundle_eids_handled,
+                    '!end' => $bundle_eids_handled + count($slice),
+                    '!total' => $num_eids));
+        }
+        $bundle_eids_handled += count($slice);
+
+        // If the bundle is from a remote site then call the appropriate
+        // function, otherwise, call the local function.
+        if ($site_id and module_exists('tripal_ws')) {
+          $entities = tripal_load_remote_entities($slice, $site_id, $bundle_name, $field_ids);
+        }
+        else {
+          $entities = tripal_load_entity('TripalEntity', $slice, FALSE, $field_ids, FALSE);
+        }
+        $job->logMessage('Got !count entities.', array('!count' => count($entities)));
+
+        // Now write each entity one at a time to the files.
+        foreach ($entities as $entity_id => $entity) {
+
+          // Write the same entity to all the formatters that are supported.
+          foreach ($formatters as $class => $formatter) {
+            //if ($class == 'TripalTabDownloader') {
+              $formatter->writeEntity($entity, $job);
+            //}
           }
         }
+        if ($job) {
+          $job->setItemsHandled($num_handled + count($slice));
+        }
       }
     }
 
-    $downloader = new $formatter($this->bundle_name, $this->ids, $supported_fields, $outfile, $this->user->uid);
-    $downloader->write();
-
+    // Now close up all the files
+    foreach ($formatters as $class => $formatter) {
+      $formatter->writeDone($job);
+    }
   }
-}
+
+
+}

tripal/includes/TripalEntityController.inc

@@ -322,7 +322,7 @@ class TripalEntityController extends EntityAPIController {
    * @return
    *   The saved entity object with updated properties.
    */
-  public function save($entity) {
+  public function save($entity, DatabaseTransaction $transaction = NULL) {
     global $user;
     $pkeys = array();
 
@@ -348,7 +348,7 @@ class TripalEntityController extends EntityAPIController {
       }
     }
 
-    $transaction  = db_transaction();
+    $transaction = isset($transaction) ? $transaction : db_transaction();
     try {
       // If our entity has no id, then we need to give it a
       // time of creation.
@@ -437,8 +437,13 @@ class TripalEntityController extends EntityAPIController {
    *  The list of key/value filters for querying the entity.
    * @param $field_ids
    *  The list of numeric field IDs for fields that should be attached.
+   * @param $cache
+   *  When loading of entities they can be cached with Drupal for later
+   *  faster loading. However, this can cause memory issues when running
+   *  Tripal jobs that load lots of entities.  Caching of entities can
+   *  be disabled to improve memory performance by setting this to FALSE.
    */
-  public function load($ids = array(), $conditions = array(), $field_ids = array()) {
+  public function load($ids = array(), $conditions = array(), $field_ids = array(), $cache = TRUE) {
 
     $entities = array();
 
@@ -529,10 +534,10 @@ class TripalEntityController extends EntityAPIController {
       EntityCacheControllerHelper::entityCacheSet($this, $queried_entities);
     }
 
-    if ($this->cache) {
+    if ($this->cache and $cache) {
       // Add entities to the cache if we are not loading a revision.
       if (!empty($queried_entities) && !$revision_id) {
-        $this->cacheSet($queried_entities);
+          $this->cacheSet($queried_entities);
 
         // Remember if we have cached all entities now.
         if (!$conditions && $ids === FALSE) {

tripal/includes/TripalFieldDownloaders/TripalCSVDownloader.inc

@@ -2,51 +2,91 @@
 
 class TripalCSVDownloader extends TripalFieldDownloader {
   /**
-   * Sets the label shown to the user describing this formatter.
+   * Sets the label shown to the user describing this formatter.  It
+   * should be a short identifier. Use the $full_label for a more
+   * descriptive label.
    */
-  static public $label = 'CSV (comma separated)';
+  static public $label = 'CSV';
+
+  /**
+   * A more verbose label that better describes the formatter.
+   */
+  static public $full_label = 'Comma separated';
 
   /**
    * Indicates the default extension for the outputfile.
    */
   static public $default_extension = 'csv';
 
+  /**
+   * @see TripalFieldDownloader::isFieldSupported()
+   */
+  public function isFieldSupported($field, $instance) {
+    $is_supported = parent::isFieldSupported($field, $instance);
+
+    // For now all fields are supported.
+    return TRUE;
+  }
+
   /**
    * @see TripalFieldDownloader::format()
    */
   protected function formatEntity($entity) {
-    $row = array();
-    foreach ($this->fields as $field_id) {
-      $field = field_info_field_by_id($field_id);
-      $field_name = $field['field_name'];
+    $bundle_name = $entity->bundle;
+    $site = !property_exists($entity, 'site_id') ? 'local' : $entity->site_id;
+    $col = array();
 
-      if (!property_exists($entity, $field_name)) {
+    // Iterate through all of the printable fields and add the value to
+    // the row.
+    foreach ($this->printable_fields as $accession => $label) {
+
+      // If this field is not present for this entity then add an empty
+      // element and move on.
+      if (!array_key_exists($accession, $this->fields2terms[$site][$bundle_name]['by_accession'])) {
+        $col[] = '';
         continue;
       }
 
-      // If we only have one element then this is good.
+      // Get the field from the class variables.
+      $field_id = $this->fields2terms[$site][$bundle_name]['by_accession'][$accession];
+      $field = $this->fields[$site][$bundle_name][$field_id]['field'];
+      $instance = $this->fields[$site][$bundle_name][$field_id]['instance'];
+      $field_name = $field['field_name'];
+
+      // If we only have one item for this value then add it.
       if (count($entity->{$field_name}['und']) == 1) {
         $value = $entity->{$field_name}['und'][0]['value'];
+
         // If the single element is not an array then this is good.
         if (!is_array($value)) {
-          $row[] = '"' . $value . '"';
+          if (is_numeric($value) or !$value) {
+            $col[] = $value;
+          }
+          else {
+            $col[] = '"' . $value . '"';
+          }
         }
         else {
-          if (array_key_exists('rdfs:label', $entity->{$field_name}['und'][0]['value'])) {
-            $row[] = strip_tags($entity->{$field_name}['und'][0]['value']['rdfs:label']);
+          if (array_key_exists('rdfs:label', $value)) {
+            $col[] = '"' . strip_tags($entity->{$field_name}['und'][0]['value']['rdfs:label']) . '"';
+          }
+          elseif (array_key_exists('schema:description', $value)) {
+            $label = $entity->{$field_name}['und'][0]['value']['schema:description'];
+            $col[] = strip_tags($label);
           }
           else {
-            $row[] = '';
+            $col[] = '';
           }
           // TODO: What to do with fields that are arrays?
         }
       }
+      // If we have multiple items then deal with that.
       else {
-        $row[] = '';
+        $col[] = '';
         // TODO: What to do with fields that have multiple values?
       }
     }
-    return array(implode(',', $row));
+    return array(implode(",", $col));
   }
 
   /**
@@ -54,12 +94,10 @@ class TripalCSVDownloader extends TripalFieldDownloader {
    */
   protected function getHeader() {
     $row = array();
-    foreach ($this->fields as $field_id) {
-      $field = field_info_field_by_id($field_id);
-      $field_name = $field['field_name'];
-      $instance = field_info_instance('TripalEntity', $field_name, $this->bundle_name);
-      $row[] = '"' . $instance['label'] . '"';
+    foreach ($this->printable_fields as $accession => $label) {
+      $row[] = $label;
     }
-    return array(implode(',', $row));
+    return array(implode(",", $row));
   }
-}
+
+}

tripal/includes/TripalFieldDownloaders/TripalFieldDownloader.inc

@@ -4,67 +4,163 @@
 abstract class TripalFieldDownloader {
 
   /**
-   * Sets the label shown to the user describing this formatter.
+   * Sets the label shown to the user describing this formatter.  It
+   * should be a short identifier. Use the $full_label for a more
+   * descriptive label.
    */
   static public $label = 'Generic';
 
+  /**
+   * A more verbose label that better describes the formatter.
+   */
+  static public $full_label = 'Generic File format';
+
   /**
    * Indicates the default extension for the outputfile.
    */
   static public $default_extension = 'txt';
 
   /**
-   * The bundle name.
+   * The data collection assigned to this downloader.
    */
-  protected $bundle_name = '';
+  protected $collection = NULL;
 
   /**
-   * A set of entity IDs. The entities must all be of the same bundle type.
+   * The collection ID
    */
-  protected $entity_ids = array();
+  protected $collection_id = NULL;
 
   /**
-   * The set of fields
+   * An array of collection_bundle records for the content types that
+   * belong to this data collection.
    */
-  protected $fields = array();
+  protected $collection_bundles = NULL;
 
   /**
    * The output file URI.
    */
   protected $outfile = '';
 
+  /**
+   * An array of printable fields.  Because fields can come from multiple
+   * bundles and those bundles can be from multiple sites, it is possible that
+   * 1) two bundles use the same field and we want to conslidate to a
+   * single printable field; and 2) that a remote site may use the same term
+   * for a field as a bundle on the local site.  The only way to sort out this
+   * mess is to use the term accession numbers.  Therefore, the array contains
+   * a unique list of printable fields using their accession numbers as keys
+   * and a field label as the value.
+   *
+   */
+  protected $printable_fields = array();
+
+  /**
+   * The remote site json data returned for the entity
+   */
+  protected $remote_entity = '';
+
+  /**
+   * An array that associates a field ID with a term.
+   *
+   * The first-level key is the site ID. For the local site this will be
+   * the word 'local' for all others it will be the numeric id.  The second
+   * level key is the bundle bundle name.  For local bundles this will
+   * always be bio_data_xxxx.  Third, are two subkeys: by_field and
+   * by_accession.  To lookup a field's term you use the 'by_field' subkey
+   * with the field_id as the next level.  To lookup the field ID for a term
+   * use the 'by_accession' subkey with the accession as the next level.  Below
+   * is an example of the structure of this array.
+   *
+   * @code
+    Array (
+      [local] => Array(
+        [bio_data_7] => Array(
+          [by_field] => Array(
+            [56] => data:2091,
+            [57] => OBI:0100026,
+            [17] => schema:name,
+            [58] => data:2044,
+            [34] => data:0842,
+            [67] => schema:alternateName,
+          ),
+          [by_accession] => Array (
+            [data:2091] => 56,
+            [OBI:0100026] => 57,
+            [schema:name] => 17,
+            [data:2044] => 58,
+            [data:0842] => 34,
+            [schema:alternateName] => 67,
+          ),
+        ),
+      ),
+    )
+   * @endcode
+   */
+  protected $fields2terms = array();
+
+
+  /**
+   * A list of field and instance items, indexed first by site_id with 'local'
+   * being the key for local fields and the numeric site_id for remote
+   * fields.  The second-levle key is the bundle_name and the the field_id.
+   * Below the field_id are the keys 'field' or 'instance' where the
+   * value of each is the field or instance details respectively.
+   */
+  protected $fields = array();
+
+
+  /**
+   * The file handle for an opeend file using during writing.
+   */
+  protected $fh;
+
   /**
    * Constructs a new instance of the TripalFieldDownloader class.
-   * @param $bundle_name
-   *   The name of the bundle to which the IDs in the $id argument belong.
-   * @param $ids
-   *   An array of entity IDs.  The order of the IDs will be the order that
-   *   output is generated.
-   * @param $fields
-   *   An array of numeric field IDs to use when constructing the download. If
-   *   no fields are provided then all fields that are appropriate for the
-   *   given type will be used.
+   *
+   * @param $collection_id
+   *   The ID for the collection.
    * @param $outfile_name
    *   The name of the output file to create. The name should not include
    *   a path.
    */
-  public function __construct($bundle_name, $ids, $fields = array(),
-      $outfile_name, $uid) {
+  public function __construct($collection_id, $outfile_name) {
 
-    $user = user_load($uid);
-    if (!$user) {
-      throw new Exception(t("The provided user ID does not reference a real user: '@uid'.", array('@uid' => $uid)));
-    }
     if (!$outfile_name) {
       throw new Exception("Please provide an outputfilename");
     }
 
-    $this->bundle_name = $bundle_name;
-    $this->entity_ids = $ids;
-    $this->fields = $fields;
+    // Get the collection record and store it.
+    $collection = db_select('tripal_collection', 'tc')
+      ->fields('tc')
+      ->condition('collection_id', $collection_id, '=')
+      ->execute()
+      ->fetchObject();
+
+    if (!$collection) {
+      throw new Exception(t("Cannot find a collection that matches the provided id: !id", array('!id' => $collection_id)));
+    }
+    $this->collection = $collection;
 
     // Make sure the user directory exists
+    $user = user_load($this->collection->uid);
     $user_dir = 'public://tripal/users/' . $user->uid;
+
+    // Set the collection ID of the collection that this downloader will use.
+    $this->collection_id = $collection_id;
+    $this->outfile = $user_dir . '/' . $outfile_name;
+
+    // A data collection may have multiple bundles.  We'll need to get
+    // them all and store them.
+    $collection_bundles = db_select('tripal_collection_bundle')
+      ->fields('tripal_collection_bundle')
+      ->condition('collection_id', $collection_id, '=')
+      ->execute();
+    while ($collection_bundle = $collection_bundles->fetchObject()) {
+      $collection_bundle->ids = unserialize($collection_bundle->ids);
+      $collection_bundle->fields = unserialize($collection_bundle->fields);
+      $this->collection_bundles[] = $collection_bundle;
+    }
+
     if (!file_prepare_directory($user_dir, FILE_CREATE_DIRECTORY)) {
       $message = 'Could not access the directory on the server for storing this file.';
       watchdog('tripal', $message, array(), WATCHDOG_ERROR);
@@ -76,7 +172,37 @@ abstract class TripalFieldDownloader {
       return;
     }
 
-    $this->outfile = $user_dir. '/' . $outfile_name;
+    // Map the fields to their term accessions.
+    $this->setFields();
+    $this->setFields2Terms();
+    $this->setPrintableFields();
+  }
+
+  /**
+   * Inidcates if a given field is supported by this Downloader class.
+   *
+   * @param $field
+   *   A field info array.
+   */
+  public function isFieldSupported($field, $instance) {
+    $field_name = $field['field_name'];
+    $field_type = $field['type'];
+    $this_formatter = get_class($this);
+
+    // If a field is a TripalField then check its supported downloaders.
+    if (tripal_load_include_field_class($field_type)) {
+      $formatters = $field_type::$download_formatters;
+      if (in_array($this_formatter, $formatters)) {
+        return TRUE;
+      }
+    }
+
+    // If this is a remote field then check that differently.
+    if ($field['storage']['type'] == 'tripal_remote_field' ) {
+      if (in_array($this_formatter, $instance['formatters'])) {
+        return TRUE;
+      }
+    }
   }
 
   /**
@@ -91,45 +217,65 @@ abstract class TripalFieldDownloader {
    */
   public function delete() {
     $fid = db_select('file_managed', 'fm')
-     ->fields('fm', array('fid'))
-     ->condition('uri', $this->outfile)
-     ->execute()
-     ->fetchField();
-     if ($fid) {
-       $file = file_load($fid);
-       file_usage_delete($file, 'tripal', 'data-collection');
-       file_delete($file, TRUE);
-     }
+      ->fields('fm', array('fid'))
+      ->condition('uri', $this->outfile)
+      ->execute()
+      ->fetchField();
+    if ($fid) {
+      $file = file_load($fid);
+      file_usage_delete($file, 'tripal', 'data-collection');
+      file_delete($file, TRUE);
+    }
   }
 
   /**
-   * Creates the downloadable file.
+   *
+   * @param TripalJob $job
    */
-  public function write() {
-    global $user;
+  public function writeInit(TripalJob $job = NULL) {
+
+    $user = user_load($this->collection->uid);
 
-    $fh = fopen(drupal_realpath($this->outfile), "w");
-    if (!$fh) {
+    $this->fh = fopen(drupal_realpath($this->outfile), "w");
+    if (!$this->fh) {
       throw new Exception("Cannout open collection file: " . $this->outfile);
     }
 
+    // Add the headers to the file.
     $headers = $this->getHeader();
     if ($headers) {
       foreach ($headers as $line) {
-        fwrite($fh, $line . "\r\n");
+        fwrite($this->fh, $line . "\r\n");
       }
     }
+  }
 
-    foreach ($this->entity_ids as $entity_id) {
-      $result = tripal_load_entity('TripalEntity', array($entity_id), FALSE, $this->fields);
-      reset($result);
-      $entity = $result[$entity_id];
-      $lines = $this->formatEntity($entity);
-      foreach ($lines as $line) {
-        fwrite($fh, $line . "\r\n");
-      }
+  /**
+   * Write a single entity to the file.
+   *
+   * Before calling this function call the initWrite() function to
+   * establish the file and write headers.
+   *
+   * @param $entity
+   *   The Entity to write.
+   * @param TripalJob $job
+   */
+  public function writeEntity($entity, TripalJob $job = NULL){
+    $lines = $this->formatEntity($entity);
+    foreach ($lines as $line) {
+      fwrite($this->fh, $line . "\r\n");
     }
-    fclose($fh);
+  }
+
+  /**
+   * Closes the output file once writing of all entities is completed.
+   *
+   * @param TripalJob $job
+   */
+  public function writeDone(TripalJob $job = NULL) {
+    fclose($this->fh);
+
+    $user = user_load($this->collection->uid);
 
     $file = new stdClass();
     $file->uri = $this->outfile;
@@ -137,12 +283,86 @@ abstract class TripalFieldDownloader {
     $file->filemime = file_get_mimetype($this->outfile);
     $file->uid = $user->uid;
     $file->status = FILE_STATUS_PERMANENT;
-    $file = file_save($file);
-    $fid = $file->fid;
-    $file = file_load($fid);
+
+    // Check if this file already exists. If it does then just update
+    // the stats.
+    $fid = db_select('file_managed', 'fm')
+      ->fields('fm', array('fid'))
+      ->condition('uri', $this->outfile)
+      ->execute()
+      ->fetchField();
+    if ($fid) {
+      $file->fid = $fid;
+      $file = file_save($file);
+    }
+    else {
+      $file = file_save($file);
+      $fid = $file->fid;
+      $file = file_load($fid);
+    }
+
     // We use the fid for the last argument because these files
-    // aren't really associated with any entity, but we need a value.
-    file_usage_add($file, 'tripal', 'data-collection', $fid);
+    // aren't really associated with any entity, but we need a value./
+    // But, only add the usage if it doens't already exists.
+    $usage = file_usage_list($file);
+    if (array_key_exists('tripal', $usage)) {
+      if (!array_key_exists('data-collection', $usage['tripal'])) {
+        file_usage_add($file, 'tripal', 'data-collection', $fid);
+      }
+    }
+  }
+
+
+  /**
+   * Creates the downloadable file.
+   *
+   * @param $job
+   *    If this function is run as a Tripal Job then this argument can be
+   *    set to the Tripaljob object for keeping track of progress.
+   */
+  public function write(TripalJob $job = NULL) {
+
+    $this->initWrite($job);
+
+    $num_handled = 0;
+    foreach ($this->collection_bundles as $bundle_collection) {
+      $collection_bundle_id = $bundle_collection->collection_bundle_id;
+      $bundle_name = $bundle_collection->bundle_name;
+      $entity_ids = $bundle_collection->ids;
+      $fields = $bundle_collection->fields;
+      $site_id = $bundle_collection->site_id;
+
+      foreach ($entity_ids as $entity_id) {
+        $num_handled++;
+        if ($job) {
+          $job->setItemsHandled($num_handled);
+        }
+
+        // if we have a site_id then we need to get the entity from the
+        // remote service. Otherwise create the entity from the local system.
+        if ($site_id) {
+          $entity = $this->loadRemoteEntity($entity_id, $site_id, $bundle_name);
+          if (!$entity) {
+            continue;
+          }
+        }
+        else {
+          $result = tripal_load_entity('TripalEntity', array($entity_id), FALSE, $fields, FALSE);
+          $entity = $result[$entity_id];
+        }
+
+        if (!$entity) {
+          continue;
+        }
+
+         $lines = $this->formatEntity($entity);
+         foreach ($lines as $line) {
+           fwrite($this->fh, $line . "\r\n");
+         }
+      }
+    }
+
+    $this->finishWrite($job);
   }
 
   /**
@@ -152,6 +372,114 @@ abstract class TripalFieldDownloader {
 
   }
 
+
+  /**
+   * A helper function for the setFields() function.
+   *
+   * Adds local fields to the list of fields.
+   */
+  private function setLocalFields() {
+    foreach ($this->collection_bundles as $collection_bundle) {
+      $bundle_name = $collection_bundle->bundle_name;
+      if ($collection_bundle->site_id) {
+        continue;
+      }
+      foreach ($collection_bundle->fields as $field_id) {
+        $field = field_info_field_by_id($field_id);
+        $instance = field_info_instance('TripalEntity', $field['field_name'], $bundle_name);
+        $this->fields['local'][$bundle_name][$field_id]['field'] = $field;
+        $this->fields['local'][$bundle_name][$field_id]['instance'] = $instance;
+      }
+    }
+  }
+
+  /**
+   * A helper function for the setFields() function.
+   *
+   * Adds remote fields to the list of fields.
+   */
+  private function setRemoteFields() {
+    // We can't use the Tripal ws API extensions if the
+    // tripal_ws module is not enabled.
+    if (!module_exists('tripal_ws')) {
+      return;
+    }
+
+    foreach ($this->collection_bundles as $collection_bundle) {
+      $bundle_name = $collection_bundle->bundle_name;
+      $site_id = $collection_bundle->site_id;
+      // Skip local fields.
+      if (!$site_id) {
+        continue;
+      }
+
+      // Iterate through the fields of this collection and get the
+      // info for each one from the class.  We will create "fake" field and
+      // instance info arrays.
+      foreach ($collection_bundle->fields as $field_id) {
+        $field = tripal_get_remote_field_info($site_id, $bundle_name, $field_id);
+        $instance = tripal_get_remote_field_instance_info($site_id, $bundle_name, $field_id);
+        $this->fields[$site_id][$bundle_name][$field_id]['field'] = $field;
+        $this->fields[$site_id][$bundle_name][$field_id]['instance'] = $instance;
+      }
+    }
+  }
+
+  /**
+   * Sets the fields array
+   */
+  protected function setFields() {
+    $this->setLocalFields();
+    $this->setRemoteFields();
+  }
+
+  /**
+   * Sets the fields2term array.
+   *
+   * The fields2term array provides an easy lookup for mapping a term
+   * to it's accession number.
+   **/
+  protected function setFields2Terms() {
+
+    foreach ($this->fields as $site => $bundles) {
+      foreach ($bundles as $bundle_name => $bundle_fields) {
+        foreach ($bundle_fields as $field_id => $info) {
+          $instance = $info['instance'];
+          $accession = $instance['settings']['term_vocabulary'] . ':' . $instance['settings']['term_accession'];
+          $this->fields2terms[$site][$bundle_name]['by_field'][$field_id] = $accession;
+          $this->fields2terms[$site][$bundle_name]['by_accession'][$accession] = $field_id;
+        }
+      }
+    }
+  }
+
+  /**
+   * Conslidates all the fields into a single list of accession numbers.
+   *
+   * The array of printable fields will contain an array containing the
+   * accession number and the label.  The title used is from the first
+   * occurance of an accession.
+   */
+  protected function setPrintableFields() {
+
+    foreach ($this->fields as $site => $bundles) {
+      foreach ($bundles as $bundle_name => $bundle_fields) {
+        foreach ($bundle_fields as $field_id => $info) {
+          $field = $info['field'];
+          $instance = $info['instance'];
+          $accession = $instance['settings']['term_vocabulary'] . ':' . $instance['settings']['term_accession'];
+          if (!array_key_exists($accession, $this->printable_fields)) {
+            // Only include fields that support this downloader type in
+            // or list of printable fields.
+            if ($this->isFieldSupported($field, $instance)) {
+              $this->printable_fields[$accession] = $instance['label'];
+            }
+          }
+        }
+      }
+    }
+  }
+
   /**
    * Formats the entity and the specified fields for output.
    *
@@ -175,4 +503,5 @@ abstract class TripalFieldDownloader {
    *  the header lines for an output file.
    */
   abstract protected function getHeader();
+
 }

tripal/includes/TripalFieldDownloaders/TripalNucFASTADownloader.inc

@@ -3,9 +3,16 @@
 class TripalNucFASTADownloader extends TripalFieldDownloader {
 
   /**
-   * Sets the label shown to the user describing this formatter.
+   * Sets the label shown to the user describing this formatter.  It
+   * should be a short identifier. Use the $full_label for a more
+   * descriptive label.
    */
-  static public $label = 'Nucleotide FASTA';
+  static public $label = 'FASTA';
+
+  /**
+   * A more verbose label that better describes the formatter.
+   */
+  static public $full_label = 'Nucleotide FASTA';
 
   /**
    * Indicates the default extension for the outputfile.
@@ -17,92 +24,116 @@ class TripalNucFASTADownloader extends TripalFieldDownloader {
    */
   protected function formatEntity($entity) {
     $lines = array();
+    $site = !property_exists($entity, 'site_id') ? 'local' : $entity->site_id;
+    $bundle_name = $entity->bundle;
 
-    // Get the list of all fields that have been attached to the entity
-    $instances = field_info_instances('TripalEntity', $entity->bundle);
-    $available_fields = array();
-    foreach ($instances as $field_name => $instance) {
-      if ($instance['field_name'] == 'entity_id') {
-        continue;
-      }
-      $available_fields[$instance['field_name']] = $instance;
-    }
+    // Holds the list of sequence identifiers that will be used to build the
+    // definition line.
+    $identifiers = array(
+      'identifier' => '',
+      'name' => '',
+      'accession' => '',
+    );
+    // Holds the list of non identifiers that will be used in the definitino
+    // line.
+    $others = array();
+    // Holds the sequence string for the FASTA item.
+    $residues = '';
 
-    foreach ($this->fields as $field_id) {
-      $field = field_info_field_by_id($field_id);
+    // Iterate through all of the fields and build the definition line and
+    // the sequence string.
+    foreach ($this->fields[$site][$bundle_name] as $field_id => $info) {
+      $field = $info['field'];
+      $instance = $info['instance'];
       $field_name = $field['field_name'];
+      $accession = $instance['settings']['term_vocabulary'] . ':' . $instance['settings']['term_accession'];
 
+      // If this field really is not attched to the entity then skip it.
       if (!property_exists($entity, $field_name)) {
         continue;
       }
 
       // If we only have one element then this is good.
       if (count($entity->{$field_name}['und']) == 1) {
+
         $value = $entity->{$field_name}['und'][0]['value'];
-        // If the single element is not an array then this is good.
-        if (!is_array($value)) {
-
-          // We need to make sure we have some fields for the definition line.
-          // those may or may not have been included, so we should add them.
-          $defline = '>';
-          $found_identifier = FALSE;
-          if (property_exists($entity, 'data__identifier')) {
-            $found_identifier = TRUE;
-            $defline .= $entity->{'data__identifier'}['und'][0]['value'] . ' ';
-          }
-          if (property_exists($entity, 'schema__name')) {
-            $found_identifier = TRUE;
-            $defline .= $entity->{'schema__name'}['und'][0]['value'] . ' ';
-          }
-          if (property_exists($entity, 'data__accession')) {
-            $found_identifier = TRUE;
-            $defline .= $entity->{'data__accession'}['und'][0]['value'] . ' ';
-          }
-          if (!$found_identifier) {
-            $defline .= "Unknown feature identifier. Please add a name field to the data collection";
-          }
-          if (property_exists($entity, 'data__sequence_coordinates')) {
-            $location = strip_tags(drupal_render(field_view_field('TripalEntity', $entity, 'data__sequence_coordinates'))) . '; ';
-            $location = preg_replace('/\&nbsp\;/', ' ', $location);
-            $defline .= $location;
-          }
-          // Add to the defnition line values from any single valued fields.
-          foreach ($available_fields as $fname => $instance) {
-            if (count($entity->{$fname}['und']) == 1) {
-              if (!is_array($entity->{$fname}['und'][0]['value'])) {
-                // Skip the identifier fields and the residues fields.
-                if (!in_array($fname, array('data__identifier',
-                    'schema__name', 'data__protein_sequence', $field_name))) {
-                  $fvalue = $entity->{$fname}['und'][0]['value'];
-                  if ($fvalue) {
-                    $defline .= $instance['label'] . ': ' . $fvalue . '; ';
-                  }
-                }
-              }
-              else {
-                if (array_key_exists('rdfs:label', $entity->{$fname}['und'][0]['value'])) {
-                  $defline .= $instance['label'] . ': ' . strip_tags($entity->{$fname}['und'][0]['value']['rdfs:label']) . '; ';
-                }
-              }
-            }
-          }
-          $defline = rtrim($defline, '; ');
 
-          // Now add the residues.
-          $lines[] = $defline;
-          $residues = explode('|', wordwrap($value, 50, "|", TRUE));
-          foreach ($residues as $line) {
-            $lines[] = $line;
-          }
+        // Add in the unique identifier for this sequence to the defline.
+        if ($accession == 'data:0842') {
+          $identifiers['identifier'] = $value;
+        }
+        // Add in the non-unique name for this sequence to the defline.
+        else if ($accession == 'schema:name') {
+          $identifiers['name'] = $value;
+        }
+        // Add in the local site accession for this sequence to the defline.
+        else if ($accession == 'data:2091') {
+          $identifiers['accession'] = $value;
+        }
+        // Add in the sequence coordinataes to the defline.
+        else if ($accession == 'data:2012') {
+          $others[$instance['label']] =  $value["data:3002"] . ':' . $value["local:fmin"] . '-' . $value["local:fmax"] . $value["data:0853"];
         }
+        // Get the nuclotide sequence.
+        else if ($accession == 'data:2044') {
+          $residues = $entity->{$field_name}['und'][0]['value'];
+        }
+        // Skip the protein sequence if it exists.
+        else if ($accession == 'data:2976') {
+          // do nothing.
+        }
+        // Add in the organism.
+        else if ($accession == 'OBI:0100026') {
+          //$others[$instance['label']] = strip_tags($value['rdfs:label']);
+        }
+        // All other fields add them to the others list.
         else {
-          // TODO: What to do with fields that are arrays?
+          if (!is_array($value)) {
+            $others[$instance['label']] = $value;
+          }
+          else {
+            // TODO: What to do with fields that are arrays?
+          }
         }
       }
       else {
         // TODO: What to do with fields that have multiple values?
       }
     }
+
+    // First add the definition line.
+    if (count(array_keys($identifiers)) == 0) {
+      $defline = ">Unknown feature identifier. The data collection must have a name or accession field";
+      $lines[] = $defline;
+    }
+    else {
+      $defline = ">";
+      $defline .= $identifiers['identifier'] ?  $identifiers['identifier'] . ' ' : '';
+      $defline .= $identifiers['name'] ?  $identifiers['name'] . ' ' : '';
+      $defline .= $identifiers['accession'] ?  $identifiers['accession'] . ' ' : '';
+
+      foreach ($others as $k => $v) {
+        if ($v) {
+          // If the value has non alpha-numeric characters then wrap it in
+          // quotes.
+          if (preg_match('/[^\w]/', $v)) {
+            $defline .= $k . ':"' . $v . '"; ';
+          }
+          else {
+            $defline .= $k . ':' . $v . '; ';
+          }
+        }
+      }
+      $lines[] = $defline;
+    }
+
+    // Now add the residues.
+    if ($residues) {
+      $sequence = explode('|', wordwrap($residues, 50, "|", TRUE));
+      foreach ($sequence as $line) {
+        $lines[] = $line;
+      }
+    }
     return $lines;
   }
 

tripal/includes/TripalFieldDownloaders/TripalProteinFASTADownloader.inc

@@ -2,10 +2,16 @@
 class TripalProteinFASTADownloader extends TripalFieldDownloader {
 
   /**
-   * Sets the label shown to the user describing this formatter.
+   * Sets the label shown to the user describing this formatter.  It
+   * should be a short identifier. Use the $full_label for a more
+   * descriptive label.
    */
-  static public $label = 'Protein FASTA';
+  static public $label = 'FASTA';
 
+  /**
+   * A more verbose label that better describes the formatter.
+   */
+  static public $full_label = 'Protein FASTA';
   /**
    * Indicates the default extension for the outputfile.
    */
@@ -16,92 +22,116 @@ class TripalProteinFASTADownloader extends TripalFieldDownloader {
    */
   protected function formatEntity($entity) {
     $lines = array();
+    $site = !property_exists($entity, 'site_id') ? 'local' : $entity->site_id;
+    $bundle_name = $entity->bundle;
 
-    // Get the list of all fields that have been attached to the entity
-    $instances = field_info_instances('TripalEntity', $entity->bundle);
-    $available_fields = array();
-    foreach ($instances as $field_name => $instance) {
-      if ($instance['field_name'] == 'entity_id') {
-        continue;
-      }
-      $available_fields[$instance['field_name']] = $instance;
-    }
+    // Holds the list of sequence identifiers that will be used to build the
+    // definition line.
+    $identifiers = array(
+      'identifier' => '',
+      'name' => '',
+      'accession' => '',
+    );
+    // Holds the list of non identifiers that will be used in the definitino
+    // line.
+    $others = array();
+    // Holds the sequence string for the FASTA item.
+    $residues = '';
 
-    foreach ($this->fields as $field_id) {
-      $field = field_info_field_by_id($field_id);
+    // Iterate through all of the fields and build the definition line and
+    // the sequence string.
+    foreach ($this->fields[$site][$bundle_name] as $field_id => $info) {
+      $field = $info['field'];
+      $instance = $info['instance'];
       $field_name = $field['field_name'];
+      $accession = $instance['settings']['term_vocabulary'] . ':' . $instance['settings']['term_accession'];
 
+      // If this field really is not attched to the entity then skip it.
       if (!property_exists($entity, $field_name)) {
         continue;
       }
 
       // If we only have one element then this is good.
       if (count($entity->{$field_name}['und']) == 1) {
-        $value = $entity->{$field_name}['und'][0]['value'];
-        // If the single element is not an array then this is good.
-        if (!is_array($value)) {
 
-          // We need to make sure we have some fields for the definition line.
-          // those may or may not have been included, so we should add them.
-          $defline = '>';
-          $found_identifier = FALSE;
-          if (property_exists($entity, 'data__identifier')) {
-            $found_identifier = TRUE;
-            $defline .= $entity->{'data__identifier'}['und'][0]['value'] . ' ';
-          }
-          if (property_exists($entity, 'schema__name')) {
-            $found_identifier = TRUE;
-            $defline .= $entity->{'schema__name'}['und'][0]['value'] . ' ';
-          }
-          if (property_exists($entity, 'data__accession')) {
-            $found_identifier = TRUE;
-            $defline .= $entity->{'data__accession'}['und'][0]['value'] . ' ';
-          }
-          if (!$found_identifier) {
-            $defline .= "Unknown feature identifier. Please add a name field to the data collection";
-          }
-          if (property_exists($entity, 'data__sequence_coordinates')) {
-            $location = strip_tags(drupal_render(field_view_field('TripalEntity', $entity, 'data__sequence_coordinates'))) . '; ';
-            $location = preg_replace('/\&nbsp\;/', ' ', $location);
-            $defline .= $location;
-          }
-          // Add to the defnition line values from any single valued fields.
-          foreach ($available_fields as $fname => $instance) {
-            if (count($entity->{$fname}['und']) == 1) {
-              if (!is_array($entity->{$fname}['und'][0]['value'])) {
-                // Skip the identifier fields and the residues fields.
-                if (!in_array($fname, array('data__identifier',
-                  'schema__name', 'data__sequence', $field_name))) {
-                  $fvalue = $entity->{$fname}['und'][0]['value'];
-                  if ($fvalue) {
-                    $defline .= $instance['label'] . ': ' . $fvalue . '; ';
-                  }
-                }
-              }
-              else {
-                if (array_key_exists('rdfs:label', $entity->{$fname}['und'][0]['value'])) {
-                  $defline .= $instance['label'] . ': ' . strip_tags($entity->{$fname}['und'][0]['value']['rdfs:label']) . '; ';
-                }
-              }
-            }
-          }
-          $defline = rtrim($defline, '; ');
+        $value = $entity->{$field_name}['und'][0]['value'];
 
-          // Now add the residues.
-          $lines[] = $defline;
-          $residues = explode('|', wordwrap($value, 50, "|", TRUE));
-          foreach ($residues as $line) {
-            $lines[] = $line;
-          }
+        // Add in the unique identifier for this sequence to the defline.
+        if ($accession == 'data:0842') {
+          $identifiers['identifier'] = $value;
+        }
+        // Add in the non-unique name for this sequence to the defline.
+        else if ($accession == 'schema:name') {
+          $identifiers['name'] = $value;
+        }
+        // Add in the local site accession for this sequence to the defline.
+        else if ($accession == 'data:2091') {
+          $identifiers['accession'] = $value;
+        }
+        // Add in the sequence coordinataes to the defline.
+        else if ($accession == 'data:2012') {
+          $others[$instance['label']] =  $value["data:3002"] . ':' . $value["local:fmin"] . '-' . $value["local:fmax"] . $value["data:0853"];
         }
+        // Skip the nuclotide sequence.
+        else if ($accession == 'data:2044') {
+          // do nothing.
+        }
+        // Get the protein sequence if it exists.
+        else if ($accession == 'data:2976') {
+          $residues = $entity->{$field_name}['und'][0]['value'];
+        }
+        // Add in the organism.
+        else if ($accession == 'OBI:0100026') {
+          $others[$instance['label']] = strip_tags($value['rdfs:label']);
+        }
+        // All other fields add them to the others list.
         else {
-          // TODO: What to do with fields that are arrays?
+          if (!is_array($value)) {
+            $others[$instance['label']] = $value;
+          }
+          else {
+            // TODO: What to do with fields that are arrays?
+          }
         }
       }
       else {
         // TODO: What to do with fields that have multiple values?
       }
     }
+
+    // First add the definition line.
+    if (count(array_keys($identifiers)) == 0) {
+      $defline = ">Unknown feature identifier. The data collection must have a name or accession field";
+      $lines[] = $defline;
+    }
+    else {
+      $defline = ">";
+      $defline .= $identifiers['identifier'] ?  $identifiers['identifier'] . ' ' : '';
+      $defline .= $identifiers['name'] ?  $identifiers['name'] . ' ' : '';
+      $defline .= $identifiers['accession'] ?  $identifiers['accession'] . ' ' : '';
+
+      foreach ($others as $k => $v) {
+        if ($v) {
+          // If the value has non alpha-numeric characters then wrap it in
+          // quotes.
+          if (preg_match('/[^\w]/', $v)) {
+            $defline .= $k . ':"' . $v . '"; ';
+          }
+          else {
+            $defline .= $k . ':' . $v . '; ';
+          }
+        }
+      }
+      $lines[] = $defline;
+    }
+
+    // Now add the residues.
+    if ($residues) {
+      $sequence = explode('|', wordwrap($residues, 50, "|", TRUE));
+      foreach ($sequence as $line) {
+        $lines[] = $line;
+      }
+    }
     return $lines;
   }
 

tripal/includes/TripalFieldDownloaders/TripalTabDownloader.inc

@@ -2,9 +2,16 @@
 
 class TripalTabDownloader extends TripalFieldDownloader {
   /**
-   * Sets the label shown to the user describing this formatter.
+   * Sets the label shown to the user describing this formatter.  It
+   * should be a short identifier. Use the $full_label for a more
+   * descriptive label.
    */
-  static public $label = 'Tab delimeted';
+  static public $label = 'TAB';
+
+  /**
+   * A more verbose label that better describes the formatter.
+   */
+  static public $full_label = 'Tab delimeted';
 
   /**
    * Indicates the default extension for the outputfile.
@@ -12,41 +19,70 @@ class TripalTabDownloader extends TripalFieldDownloader {
   static public $default_extension = 'txt';
 
   /**
-   * @see TripalFieldDownloader::format()
+   * @see TripalFieldDownloader::isFieldSupported()
    */
-  protected function formatEntity($entity) {
-    $row = array();
-    foreach ($this->fields as $field_id) {
-      $field = field_info_field_by_id($field_id);
-      $field_name = $field['field_name'];
+  public function isFieldSupported($field, $instance) {
+    $is_supported = parent::isFieldSupported($field, $instance);
 
-      if (!property_exists($entity, $field_name)) {
-        continue;
-      }
+    // For now all fields are supported.
+    return TRUE;
+  }
 
-      // If we only have one element then this is good.
-      if (count($entity->{$field_name}['und']) == 1) {
-        $value = $entity->{$field_name}['und'][0]['value'];
-        // If the single element is not an array then this is good.
-        if (!is_array($value)) {
-          $row[] = $value;
-        }
-        else {
-          if (array_key_exists('rdfs:label', $entity->{$field_name}['und'][0]['value'])) {
-            $row[] = strip_tags($entity->{$field_name}['und'][0]['value']['rdfs:label']);
-          }
-          else {
-            $row[] = '';
-          }
-          // TODO: What to do with fields that are arrays?
-        }
-      }
-      else {
-        $row[] = '';
-        // TODO: What to do with fields that have multiple values?
-      }
+  /**
+   * @see TripalFieldDownloader::formatEntity()
+   */
+   protected function formatEntity($entity) {
+     $bundle_name = $entity->bundle;
+     $site = !property_exists($entity, 'site_id') ? 'local' : $entity->site_id;
+     $col = array();
+
+     // Iterate through all of the printable fields and add the value to
+     // the row.
+     foreach ($this->printable_fields as $accession => $label) {
+
+       // If this field is not present for this entity then add an empty
+       // element and move on.
+       if (!array_key_exists($accession, $this->fields2terms[$site][$bundle_name]['by_accession'])) {
+         $col[] = '';
+         continue;
+       }
+
+       // Get the field from the class variables.
+       $field_id = $this->fields2terms[$site][$bundle_name]['by_accession'][$accession];
+       $field = $this->fields[$site][$bundle_name][$field_id]['field'];
+       $instance = $this->fields[$site][$bundle_name][$field_id]['instance'];
+       $field_name = $field['field_name'];
+
+       // If we only have one item for this value then add it.
+       if (count($entity->{$field_name}['und']) == 1) {
+         $value = $entity->{$field_name}['und'][0]['value'];
+
+         // If the single element is not an array then this is good.
+         if (!is_array($value)) {
+           $col[] = $value;
+         }
+         else {
+           if (array_key_exists('rdfs:label', $value)) {
+             $label = $entity->{$field_name}['und'][0]['value']['rdfs:label'];
+             $col[] = strip_tags($label);
+           }
+           elseif (array_key_exists('schema:description', $value)) {
+             $label = $entity->{$field_name}['und'][0]['value']['schema:description'];
+             $col[] = strip_tags($label);
+           }
+           else {
+             $col[] = '';
+           }
+           // TODO: What to do with fields that are arrays?
+         }
+       }
+       // If we have multiple items then deal with that.
+       else {
+         $col[] = '';
+         // TODO: What to do with fields that have multiple values?
+       }
     }
-    return array(implode("\t", $row));
+    return array(implode("\t", $col));
   }
 
   /**
@@ -54,12 +90,9 @@ class TripalTabDownloader extends TripalFieldDownloader {
    */
   protected function getHeader() {
     $row = array();
-    foreach ($this->fields as $field_id) {
-      $field = field_info_field_by_id($field_id);
-      $field_name = $field['field_name'];
-      $instance = field_info_instance('TripalEntity', $field_name, $this->bundle_name);
-      $row[] = $instance['label'];
+    foreach ($this->printable_fields as $accession => $label) {
+      $row[] = $label;
     }
     return array(implode("\t", $row));
   }
-}
+}

tripal/includes/TripalFields/TripalField.inc

@@ -50,12 +50,21 @@ class TripalField {
     // Set to TRUE if the site admin is not allowed to change the term
     // type, otherwise the admin can change the term mapped to a field.
     'term_fixed' => FALSE,
-    // Indicates the download formats for this field.  The list must be the
-    // name of a child class of the TripalFieldDownloader.
-    'download_formatters' => array(
-      'TripalTabDownloader',
-      'TripalCSVDownloader',
-    ),
+    // Set to TRUE if the field should be automatically attached to an entity
+    // when it is loaded. Otherwise, the callee must attach the field
+    // manually.  This is useful to prevent really large fields from slowing
+    // down page loads.  However, if the content type display is set to
+    // "Hide empty fields" then this has no effect as all fields must be
+    // attached to determine which are empty.  It should always work with
+    // web services.
+    'auto_attach' => TRUE,
+  );
+
+  // Indicates the download formats for this field.  The list must be the
+  // name of a child class of the TripalFieldDownloader.
+  public static $download_formatters = array(
+    'TripalTabDownloader',
+    'TripalCSVDownloader',
   );
 
   // The default widget for this field.
@@ -132,10 +141,13 @@ class TripalField {
       $this->term = tripal_get_term_details($vocabulary, $accession);
     }
     else {
-      tripal_report_error('tripal_field', TRIPAL_ERROR, 'Unable to instantiate Field :name due to missing vocabulary and/or accession.',
-        array(':name' => $class::$default_label));
+      $bundle = tripal_load_bundle_entity(array('name' => $instance['bundle']));
+      tripal_report_error('tripal_field', TRIPAL_ERROR,
+        'Unable to instantiate the field named, ":name", due to missing vocabulary and/or accession. The term provided was: ":term". The bundle is: ":bundle".',
+        array(':name' => $instance['field_name'],
+          ':term' => $vocabulary . ':' . $accession,
+          ':bundle' => $bundle->label));
     }
-
     if (!$instance) {
       tripal_set_message(t('Missing instance of field "%field"', array('%field' => $field['field_name'])), TRIPAL_ERROR);
     }

tripal/includes/TripalJob.inc

@@ -12,6 +12,35 @@ class TripalJob {
    */
   protected $job = NULL;
 
+
+  /**
+   * The number of items that this importer needs to process. A progress
+   * can be calculated by dividing the number of items process by this
+   * number.
+   */
+  private $total_items;
+
+  /**
+   * The number of items that have been handled so far.  This must never
+   * be below 0 and never exceed $total_items;
+   */
+  private $num_handled;
+
+  /**
+   * The interval when the job progress should be updated. Updating the job
+   * progress incurrs a database write which takes time and if it occurs to
+   * frequently can slow down the loader.  This should be a value between
+   * 0 and 100 to indicate a percent interval (e.g. 1 means update the
+   * progress every time the num_handled increases by 1%).
+   */
+  private $interval;
+
+  /**
+   * Each time the job progress is updated this variable gets set.  It is
+   * used to calculate if the $interval has passed for the next update.
+   */
+  private $prev_update;
+
   /**
    * Instantiates a new TripalJob object.
    *
@@ -254,9 +283,11 @@ class TripalJob {
     try {
 
       // Include the necessary files needed to run the job.
-      foreach ($this->job->includes as $path) {
-        if ($path) {
-          require_once $path;
+      if (is_array($this->job->includes)) {
+        foreach ($this->job->includes as $path) {
+          if ($path) {
+            require_once $path;
+          }
         }
       }
 
@@ -403,6 +434,81 @@ class TripalJob {
       ->condition('job_id', $this->job->job_id)
       ->execute();
   }
+
+  /**
+   * Sets the total number if items to be processed.
+   *
+   * This should typically be called near the beginning of the loading process
+   * to indicate the number of items that must be processed.
+   *
+   * @param $total_items
+   *   The total number of items to process.
+   */
+  public function setTotalItems($total_items) {
+    $this->total_items = $total_items;
+  }
+
+  /**
+   * Adds to the count of the total number of items that have been handled.
+   *
+   * @param $num_handled
+   */
+  public function addItemsHandled($num_handled) {
+    $items_handled = $this->num_handled = $this->num_handled + $num_handled;
+    $this->setItemsHandled($items_handled);
+  }
+  /**
+   * Sets the number of items that have been processed.
+   *
+   * This should be called anytime the loader wants to indicate how many
+   * items have been processed.  The amount of progress will be
+   * calculated using this number.  If the amount of items handled exceeds
+   * the interval specified then the progress is reported to the user.  If
+   * this loader is associated with a job then the job progress is also updated.
+   *
+   * @param $total_handled
+   *   The total number of items that have been processed.
+   */
+  public function setItemsHandled($total_handled) {
+    // First set the number of items handled.
+    $this->num_handled = $total_handled;
+
+    if ($total_handled == 0) {
+      $memory = number_format(memory_get_usage());
+      print "Percent complete: 0%. Memory: " . $memory . " bytes.\r";
+      return;
+    }
+
+    // Now see if we need to report to the user the percent done.  A message
+    // will be printed on the command-line if the job is run there.
+    $percent = sprintf("%.2f", ($this->num_handled / $this->total_items) * 100);
+    $diff = $percent - $this->prev_update;
+
+    if ($diff >= $this->interval) {
+
+      $memory = number_format(memory_get_usage());
+      print "Percent complete: " . $percent . "%. Memory: " . $memory . " bytes.\r";
+      $this->prev_update = $diff;
+      $this->setProgress($percent);
+    }
+  }
+
+  /**
+   * Updates the percent interval when the job progress is updated.
+   *
+   * Updating the job
+   * progress incurrs a database write which takes time and if it occurs to
+   * frequently can slow down the loader.  This should be a value between
+   * 0 and 100 to indicate a percent interval (e.g. 1 means update the
+   * progress every time the num_handled increases by 1%).
+   *
+   * @param $interval
+   *   A number between 0 and 100.
+   */
+  public function setInterval($interval) {
+    $this->interval = $interval;
+  }
+
   /**
    * Retrieves the status of the job.
    */

tripal/includes/TripalTerm.inc

@@ -40,7 +40,7 @@ class TripalTerm extends Entity {
     return $this->name;
   }
   public function getAccession() {
-    return $this->name;
+    return $this->vocab->vocabulary . ':' . $this->accession;
   }
   public function getDefinition() {
     return $this->definition;

tripal/includes/TripalTermController.inc

@@ -18,16 +18,24 @@ class TripalTermController extends EntityAPIController {
   /**
    * Delete a single entity.
    */
-  public function delete($entity) {
-    $transaction = db_transaction();
+  public function delete($ids, DatabaseTransaction $transaction = NULL) {
+    $entities = $ids ? $this->load($ids) : FALSE;
+    if (!$entities) {
+      // Do nothing, in case invalid or no ids have been passed.
+      return;
+    }
+    $transaction = isset($transaction) ? $transaction : db_transaction();
     try {
       // Invoke hook_entity_delete().
-      module_invoke_all('entity_delete', $entity, 'TripalTerm');
-      field_attach_delete('TripalTerm', $entity);
+      $ids = array_keys($entities);
+      foreach ($entities as $id => $entity) {
+        module_invoke_all('entity_delete', $entity, 'TripalTerm');
+        field_attach_delete('TripalTerm', $entity);
 
-      db_delete('tripal_term')
-        ->condition('accession', $entity->accession)
-        ->execute();
+        db_delete('tripal_term')
+          ->condition('accession', $entity->accession)
+          ->execute();
+      }
     }
     catch (Exception $e) {
       $transaction->rollback();
@@ -41,11 +49,11 @@ class TripalTermController extends EntityAPIController {
   /**
    * Saves the custom fields using drupal_write_record().
    */
-  public function save($entity) {
+  public function save($entity, DatabaseTransaction $transaction = NULL) {
     global $user;
     $pkeys = array();
 
-    $transaction  = db_transaction();
+    $transaction = isset($transaction) ? $transaction : db_transaction();
     try {
       // If our entity has no id, then we need to give it a
       // time of creation.

tripal/includes/TripalVocabController.inc

@@ -21,15 +21,23 @@ class TripalVocabController extends EntityAPIController {
    *
    * Really a convenience function for deleteMultiple().
    */
-  public function delete($entity) {
-    $transaction = db_transaction();
+  public function delete($ids, DatabaseTransaction $transaction = NULL) {
+    $entities = $ids ? $this->load($ids) : FALSE;
+    if (!$entities) {
+      // Do nothing, in case invalid or no ids have been passed.
+      return;
+    }
+    $transaction = isset($transaction) ? $transaction : db_transaction();
     try {
-      // Invoke hook_entity_delete().
-      module_invoke_all('entity_delete', $entity, 'TripalTerm');
-      field_attach_delete('TripalVocab', $entity);
+      $ids = array_keys($entities);
+      foreach ($entities as $id => $entity) {
+        // Invoke hook_entity_delete().
+        module_invoke_all('entity_delete', $entity, 'TripalTerm');
+        field_attach_delete('TripalVocab', $entity);
+      }
 
       db_delete('tripal_term')
-        ->condition('id', $entity->id)
+        ->condition('id', $ids)
         ->execute();
     }
     catch (Exception $e) {
@@ -44,11 +52,11 @@ class TripalVocabController extends EntityAPIController {
   /**
    * Saves the custom fields using drupal_write_record().
    */
-  public function save($entity) {
+  public function save($entity, DatabaseTransaction $transaction = NULL) {
     global $user;
     $pkeys = array();
 
-    $transaction  = db_transaction();
+    $transaction = isset($transaction) ? $transaction : db_transaction();
     try {
       // If our entity has no id, then we need to give it a
       // time of creation.

tripal/includes/tripal.admin.inc

@@ -21,11 +21,11 @@ function tripal_admin_data_collection_form($form, &$form_state) {
   $form['lifespan'] = array(
     '#type' => 'textfield',
     '#title' => t('Collection Lifespan'),
-    '#description' => t('Enter the number of hours (e.g. 24 is 1 day, 720 is 30 days)
-       that data collections exist.  Collections will be automatically removed after the lifespan
+    '#description' => t('Enter the number of days that data collections exist.  
+       Collections will be automatically removed after the lifespan
        period has passed.  Removal of data collections occurs when the
        sites Drupal cron executes.'),
-    '#default_value' => variable_get('tripal_data_collections_lifespan', 7),
+    '#default_value' => variable_get('tripal_data_collections_lifespan', 30),
     '#required' => TRUE,
   );
 

tripal/includes/tripal.collections.inc

@@ -7,7 +7,8 @@
 function tripal_user_collections_page() {
   global $user;
 
-  $headers = array('Name', 'Description', 'Create Date', 'Downloads Formats', 'Actions');
+  $headers = array('Name', 'Description', 'Create Date', 'Download Formats',
+    'Generate File', 'Actions');
   $rows = array();
 
   $collections = db_select('tripal_collection', 'tc')
@@ -21,11 +22,14 @@ function tripal_user_collections_page() {
     $collection->load($collection_id);
 
     $downloads = array();
-    $formatters = $collection->getDownloadFormatters();
+    $formatters = $collection->getFormatters();
+
     foreach ($formatters as $class_name => $label) {
+
       $outfile = $collection->getOutfilePath($class_name);
-      $outfileURL = file_create_url($outfile);
+
       if (file_exists($outfile)) {
+        $outfileURL = file_create_url($outfile);
         $downloads[] = l($label, $outfileURL);
       }
       else {
@@ -45,6 +49,8 @@ function tripal_user_collections_page() {
         $collection->getDescription(),
         $collection->getCreateDate(),
         $download_list,
+        l('Generate File for Download', 'user/' . $user->uid . '/data-collections/generate/' . $collection_id),
+        l('View', 'user/' . $user->uid . '/data-collections/' . $collection_id . '/view') . ' ' .
         l('Delete', 'user/' . $user->uid . '/data-collections/' . $collection_id . '/delete'),
       ),
     );
@@ -75,6 +81,98 @@ function tripal_user_collections_page() {
   return $content;
 }
 
+/**
+ * Renders a view of a data collection
+ */
+function tripal_user_collections_view_page($uid, $collection_id) {
+
+  $user = user_load($uid);
+  // set the breadcrumb
+  $breadcrumb = array();
+  $breadcrumb[] = l('Home', '<front>');
+  $breadcrumb[] = l($user->name, 'user/' . $uid);
+  $breadcrumb[] = l('Data Collections', 'user/' . $uid . '/data-collections');
+  drupal_set_breadcrumb($breadcrumb);
+
+  $collection = new TripalEntityCollection();
+  $collection->load($collection_id);
+
+  drupal_set_title('Data Collection: ' . $collection->getName());
+
+  // Get the content types for this data collection.
+  $cbundles = $collection->getBundles();
+  $content_types = array();
+  $field_list = array();
+  $formatters_list = array();
+  $num_entities = 0;
+
+  foreach ($cbundles as $cbundle) {
+    $eids = $collection->getEntityIDs($cbundle->bundle_name);
+    $fields = $collection->getFieldIDs($cbundle->bundle_name);
+
+    // Convert local field IDs to their names.
+    if (!$cbundle->site_id) {
+      $bundle = tripal_load_bundle_entity(array('name' => $cbundle->bundle_name));
+      $content_types[] = $bundle->label;
+
+      foreach ($fields as $field_id) {
+        $field = field_info_field_by_id($field_id);
+        $instance = field_info_instance('TripalEntity', $field['field_name'], $bundle->name);
+        $field_list[] = $instance['label'];
+
+        $field_formatters = tripal_get_field_field_formatters($field, $instance);
+        foreach ($field_formatters as $class_name => $label) {
+          tripal_load_include_downloader_class($class_name);
+          $formatters_list[] = $class_name::$label . ' (' . $class_name::$full_label . ')';
+        }
+      }
+    }
+    // Convert remote field IDs to their names.
+    // TODO: add in retrieval of remote details.
+
+
+    $num_entities += count($eids);
+  }
+
+
+  $contents['content_types'] = array(
+    '#type' => 'item',
+    '#title' => 'Content types',
+    '#description' => t('The content types in this data collection'),
+    '#markup' => join(', ', $content_types),
+  );
+
+  $contents['field_list'] = array(
+    '#type' => 'item',
+    '#title' => 'Fields',
+    '#description' => t('The fields listed in this data collection'),
+    '#markup' => join(', ', array_unique($field_list)),
+  );
+  $contents['formatters'] = array(
+    '#type' => 'item',
+    '#title' => 'Supported File Types',
+    '#description' => t('The fields in your data collection appear to support the above file formats.  Please note that not all fields are compatible with all formats and only those that are compatible will appear in the format. Some file formats require certain fields and if all fields are not present in the collection they will not be complete.'),
+    '#markup' => join(', ', array_unique($formatters_list)),
+  );
+  $contents['ecount'] = array(
+    '#type' => 'item',
+    '#title' => 'Records',
+    '#description' => t('The number of records contained in this data collection.'),
+    '#markup' => number_format($num_entities),
+  );
+
+  $contents['peek'] = array(
+    '#type' => 'item',
+    '#title' => 'Data View',
+    '#markup' => 'Coming soon...',
+  );
+
+
+  return $contents;
+}
+/**
+ * Provides the confirm form for deleting a data collection.
+ */
 function tripal_user_collections_delete_form($form, &$form_state,  $uid, $collection_id) {
   $form_state['collection_id'] = $collection_id;
   $form['#submit'][] = 'tripal_user_collections_delete_form_submit';
@@ -107,8 +205,49 @@ function tripal_user_collections_delete_form_submit($form, &$form_state) {
       drupal_set_message('The data collection has been deleted.');
     }
     catch (Exception $e) {
-      drupal_set_message('There was a problem deleting the data collection please contact the site to report the error.', 'error');
+      drupal_set_message(t('There was a problem deleting the data collection ' .
+          'please contact the site to report the error: !message', array('!message', $e->getMessage())), 'error');
+    }
+  }
+  drupal_goto('user/' . $user->uid . '/data-collections');
+}
+
+function tripal_user_collections_generate_file_form($form, &$form_state,  $uid, $collection_id) {
+  $form_state['collection_id'] = $collection_id;
+  $form['#submit'][] = 'tripal_user_collections_generate_file_form_submit';
+
+  $collection  = new TripalEntityCollection();
+  $collection->load($collection_id);
+  $form = confirm_form($form,
+      t('Confirm creation of files for the data collection named: "%title"',
+          array('%title' => $collection->getName())), 'user/' . $uid . '/data-collections',
+      '<p>' .t('It may take some time for the file(s) to be generated.  An email will be sent when files are ready.') .'</p>', t('Generate File'), t('Cancel'), 'confirm');
+
+  return $form;
+}
+/**
+ * Deletes a user's collection.
+ *
+ * @param $collection_id
+ *   The ID of the collection to delete.
+ */
+function tripal_user_collections_generate_file_form_submit($form, &$form_state) {
+  global $user;
+  $collection_id = $form_state['collection_id'];
+  $collection  = new TripalEntityCollection();
+  $collection->load($collection_id);
+
+  if ($collection->getUserID() == $user->uid) {
+    try {
+      // Add the job to write the collection download files.
+      $args = array($collection_id);
+      tripal_add_job('Create data collection files for ' . $user->name, 'tripal',
+      'tripal_create_collection_files', $args, $user->uid, 10, array());
+    }
+    catch (Exception $e) {
+      drupal_set_message('There was a problem creating the file for the data collection please contact the site to report the error.', 'error');
     }
   }
+
   drupal_goto('user/' . $user->uid . '/data-collections');
-}
+}

tripal/includes/tripal.fields.inc

@@ -61,15 +61,6 @@ function tripal_field_info_alter(&$info) {
       $info[$field_name]['instance_settings']['term_fixed'] = FALSE;
       $info[$field_name]['instance_settings']['auto_attach'] = TRUE;
     }
-
-    // Make sure all fields have some sort of downloader support. The Tab
-    // delimited and CSV downloaders should support all types of fields.
-    if (!array_key_exists('download_formatters', $details)) {
-      $info[$field_name]['instance_settings']['download_formatters'] = array(
-        'TripalTabDownloader',
-        'TripalCSVDownloader',
-      );
-    }
   }
 }
 

tripal/tripal.install

@@ -155,6 +155,7 @@ function tripal_schema() {
   $schema['tripal_token_formats'] = tripal_tripal_token_formats_schema();
   $schema['tripal_variables'] = tripal_tripal_variables_schema();
 
+
   // Adds a table for managing TripalEntity entities.
   $schema['tripal_vocab'] = tripal_tripal_vocab_schema();
   $schema['tripal_term'] = tripal_tripal_term_schema();
@@ -297,7 +298,44 @@ function tripal_tripal_collection_schema() {
         'type' => 'text',
         'not null' => FALSE
       ),
-      'bundle_name' => array(
+      'uid' => array(
+        'type' => 'int',
+        'not null' => TRUE,
+        'description' => 'The user Id of the person who created the collection.'
+      ),
+      'create_date' => array(
+        'type' => 'int',
+        'not null' => TRUE,
+        'description' => 'UNIX integer start time'
+      ),
+    ),
+    'indexes' => array(
+      'uid' => array('uid')
+    ),
+    'unique keys' => array(
+      'user_collection' => array('uid', 'collection_name'),
+    ),
+    'primary key' => array('collection_id'),
+  );
+}
+
+/**
+ * Returns the Drupal Schema API array for the tripal_jobs table.
+ */
+function tripal_tripal_collection_bundle_schema() {
+  return array(
+    'fields' => array(
+      'collection_bundle_id' => array(
+        'type' => 'serial',
+        'unsigned' => TRUE,
+        'not null' => TRUE
+      ),
+      'collection_id' => array(
+        'type' => 'int',
+        'unsigned' => TRUE,
+        'not null' => TRUE
+      ),
+     'bundle_name' => array(
         'type' => 'varchar',
         'length' => 1024,
         'not null' => TRUE
@@ -314,27 +352,20 @@ function tripal_tripal_collection_schema() {
         'not null' => TRUE,
         'description' => 'An array of numeric field IDs.'
       ),
-      'uid' => array(
-        'type' => 'int',
-        'not null' => TRUE,
-        'description' => 'The user Id of the person who created the collection.'
-      ),
-      'create_date' => array(
+      'site_id' => array(
         'type' => 'int',
-        'not null' => TRUE,
-        'description' => 'UNIX integer start time'
+        'size' => 'normal',
+        'not null' => FALSE,
+        'description' => 'The ID of the site from the Tripal Sites table.'
       ),
     ),
     'indexes' => array(
-      'bundle_name' => array('bundle_name'),
-      'uid' => array('uid')
+      'collection_id' => array('collection_id')
     ),
-    'unique keys' => array(
-      'user_collection' => array('uid', 'collection_name'),
-    ),
-    'primary key' => array('collection_id'),
+    'primary key' => array('collection_bundle_id'),
   );
 }
+
 /**
  * Returns the Drupal Schema API array for the tripal_jobs table.
  */
@@ -994,3 +1025,53 @@ function tripal_update_7307() {
   );
   menu_save($menu);
 }
+
+/**
+ * Remove the bundle_name, ids, fields from the tripal collections table.
+ * And add the new tripal_tripal_collection_bundle_schema
+ */
+function tripal_update_7308() {
+  $transaction = db_transaction();
+  try {
+    if (db_field_exists('tripal_collection', 'bundle_name')) {
+      db_drop_field('tripal_collection', 'bundle_name');
+    }
+    if (db_field_exists('tripal_collection', 'ids')) {
+      db_drop_field('tripal_collection', 'ids');
+    }
+    if (db_field_exists('tripal_collection', 'fields')) {
+      db_drop_field('tripal_collection', 'fields');
+    }
+     $schema = array();
+    $schema['tripal_collection_bundle'] = tripal_tripal_collection_bundle_schema();
+    db_create_table('tripal_collection_bundle', $schema['tripal_collection_bundle']);
+  }
+  catch (\PDOException $e) {
+    $transaction->rollback();
+    $error = $e->getMessage();
+    throw new DrupalUpdateException('Could not add the tripal_collection table:' . $error);
+  }
+}
+
+/**
+ * Add the site_id field to the tripal_collection_bundle table.
+ */
+function tripal_update_7309() {
+  $transaction = db_transaction();
+  try {
+    if (!db_field_exists('tripal_collection_bundle', 'site_id')) {
+       $field = array(
+        'type' => 'int',
+        'size' => 'normal',
+        'not null' => FALSE,
+        'description' => 'The ID of the site from the Tripal Sites table.',
+      );
+      db_add_field('tripal_collection_bundle', 'site_id', $field);
+    }
+  }
+  catch (\PDOException $e) {
+    $transaction->rollback();
+    $error = $e->getMessage();
+    throw new DrupalUpdateException('Could not add the tripal_collection table:' . $error);
+  }
+}

tripal/tripal.module

@@ -390,6 +390,28 @@ function tripal_menu() {
     'file' => 'includes/tripal.collections.inc',
     'file path' => drupal_get_path('module', 'tripal'),
   );
+  $items['user/%/data-collections/%/view'] = array (
+    'title' => 'View a Collections',
+    'description' => 'Views a data collection.',
+    'page callback' => 'tripal_user_collections_view_page',
+    'page arguments' => array(1, 3),
+    'access callback' => 'tripal_accesss_user_collections',
+    'access arguments' => array(1),
+    'type' => MENU_CALLBACK,
+    'file' => 'includes/tripal.collections.inc',
+    'file path' => drupal_get_path('module', 'tripal'),
+  );
+  $items['user/%/data-collections/generate/%'] = array (
+    'title' => 'Generate a file for download of a Collections',
+    'description' => 'Generates a data collection file for download.',
+    'page callback' => 'drupal_get_form',
+    'page arguments' => array('tripal_user_collections_generate_file_form', 1, 4),
+    'access callback' => 'tripal_accesss_user_collections',
+    'access arguments' => array(1),
+    'type' => MENU_CALLBACK,
+    'file' => 'includes/tripal.collections.inc',
+    'file path' => drupal_get_path('module', 'tripal'),
+  );
 
   $items['admin/tripal/data-collections'] = array(
     'title' => 'Data Collections',
@@ -532,6 +554,11 @@ function tripal_theme($existing, $type, $theme, $path) {
       'render element' => 'element',
       'file' => 'includes/tripal.fields.inc',
     ),
+    // Themed forms
+    'tripal_views_handler_area_collections_fields_fset' => array(
+      'render element' => 'form',
+      'file' => 'views_handlers/tripal_views_handler_area_collections.inc',
+    ),
   );
 
   return $themes;
@@ -747,16 +774,42 @@ function tripal_import_api() {
 }
 
 /**
- * Implements hook_form_alter().
+ *
+ * Implements hook_form_FORM_ID_alter().
+ *
+ * The field_ui_field_edit_form is used for customizing the settings of
+ * a field attached to an entity.
+ *
+ * This alter function disables some of the form widgets when the storage
+ * backend indicates they are not appropriate.
  */
-function tripal_form_alter(&$form, $form_state, $form_id) {
+function tripal_form_field_ui_field_edit_form_alter(&$form, &$form_state, $form_id) {
   // If this is the field_ui_field_edit_form (i.e. the form that appears
   // when editing a field that is attached to an entity). Then we want
   // to add term settings for any field attached to a TripalEntity
   // content type.
-  if ($form_id == 'field_ui_field_edit_form' and $form['#instance']['entity_type'] == 'TripalEntity') {
+  if ($form['#instance']['entity_type'] == 'TripalEntity') {
+
+    // Add the form elements for setting the vocabulary terms.
     tripal_field_instance_settings_form_alter($form, $form_state);
+
+    // Make sure we preserve the auto_attach settings.
+    $auto_attach = TRUE;
+    if (array_key_exists('auto_attach', $form['#instance']['settings'])) {
+      $auto_attach = $form['#instance']['settings']['auto_attach'];
+    }
+    $form['instance']['settings']['auto_attach'] = array(
+      '#type' => 'value',
+      '#value' => $auto_attach,
+    );
   }
+}
+
+
+/**
+ * Implements hook_form_alter().
+ */
+function tripal_form_alter(&$form, $form_state, $form_id) {
 
   // Remove fields that have no form. It's just a bit too confusing to have
   // widgets appear in the form but without any form elements inside them.
@@ -1059,14 +1112,30 @@ function tripal_cron() {
     $modules = module_implements('tripal_cron_notification');
     foreach ($modules as $module) {
       $function = $module . '_tripal_cron_notification';
-      tripal_add_job("Cron: Checking for '$module' notifications.", 'tripal',
-        $function, $args, 1, $includes, TRUE);
+      tripal_add_job(
+        "Cron: Checking for '$module' notifications.",    // Job Name
+        'tripal',                                         // Module Name
+        $function,                                        // Callback
+        $args,                                            // Arguements
+        1,                                                // User UID
+        1,                                                // Priority (1-10)
+        $includes,                                        // Includes
+        TRUE                                              // Ignore Duplicates
+      );
     }
   }
 
   // Check for expired collections.
-  tripal_add_job('Cron: Checking expired collections', 'tripal',
-    'tripal_expire_collections', $args, 1, $includes, TRUE);
+  tripal_add_job(
+    'Cron: Checking expired collections',             // Job Name
+    'tripal',                                         // Module Name
+    'tripal_expire_collections',                      // Callback
+    $args,                                            // Arguements
+    1,                                                // User UID
+    1,                                                // Priority (1-10)
+    $includes,                                        // Includes
+    TRUE                                              // Ignore Duplicates
+  );
 }
 
 /**

tripal/views_handlers/tripal_views_handler_area_collections.inc

@@ -46,13 +46,13 @@ function tripal_views_handler_area_collections_form($form, $form_state, $view, $
 
   $form = array();
   $form['save_collection'] = array(
-   '#type' => 'fieldset',
-   '#title' => t('Save Results'),
-   '#collapsible' => TRUE,
-   '#collapsed' => TRUE,
-   '#description' => t('A data collection is a virtual container into which you can
-     save data.  You can place your search results into a data collection for
-     download or use with other tools on this site that support data collections.'),
+    '#type' => 'fieldset',
+    '#title' => t('Save Results'),
+    '#collapsible' => TRUE,
+    '#collapsed' => TRUE,
+    '#description' => t('A data collection is a virtual container into which you can
+      save data.  You can place your search results into a data collection for
+      download or use with other tools on this site that support data collections.'),
   );
   $form['save_collection']['bundle'] = array(
     '#type' => 'value',
@@ -67,22 +67,28 @@ function tripal_views_handler_area_collections_form($form, $form_state, $view, $
     '#value' => unserialize(serialize($query->query))
   );
   $form['save_collection']['summary'] = array(
-   '#type' => 'item',
-   '#title' => 'Results Summary',
-   '#markup' => t('There are @total_rows record(s) that can be added to a data collection.', array('@total_rows' => $view->total_rows)),
+    '#type' => 'item',
+    '#title' => 'Results Summary',
+    '#markup' => t('There are @total_rows record(s) that can be added to a data collection.', array('@total_rows' => $view->total_rows)),
   );
   $form['save_collection']['collection_name'] = array(
-   '#type' => 'textfield',
-   '#title' => t('Collection Name'),
-   '#description' => t('Please name this collection for future reference.'),
-   '#default_value' => $collection_name,
-   '#required' => TRUE,
+    '#type' => 'textfield',
+    '#title' => t('Collection Name'),
+    '#description' => t('Please name this collection for future reference.'),
+    '#default_value' => $collection_name,
+    '#required' => TRUE,
   );
-  $form['save_collection']['collection_desc'] = array(
-   '#type' => 'textarea',
-   '#title' => t('Description'),
-   '#description' => t('Please provide a description about this data collection.'),
-   '#default_value' => $collection_name,
+  $form['save_collection']['description_fset'] = array(
+    '#type' => 'fieldset',
+    '#title' => t('Add a Description'),
+    '#collapsible' => TRUE,
+    '#collapsed' => TRUE,
+  );
+  $form['save_collection']['description_fset']['collection_desc'] = array(
+    '#type' => 'textarea',
+    '#title' => t('Description'),
+    '#description' => t('Please provide a description about this data collection. This is meant to help you remember what is in the collection.'),
+    '#default_value' => $collection_name,
   );
 
   // Get the list of fields used in the view.
@@ -93,6 +99,24 @@ function tripal_views_handler_area_collections_form($form, $form_state, $view, $
   else {
     $view_fields = $view->display['default']->display_options['fields'];
   }
+
+  $form['save_collection']['fields'] = array(
+    '#type' => 'fieldset',
+    '#title' => t('Add or Update Fields'),
+    '#description' => t('You may select any of the additional fields below to
+      add to this data collection. Please note that different fields may be able
+      to create different output file types.'),
+    '#collapsible' => TRUE,
+    '#collapsed' => TRUE,
+  );
+
+  // We want to theme all of the fields, so we add this next level in the
+  // form array to theme.
+  $form['save_collection']['fields']['items'] = array(
+    '#theme' => 'tripal_views_handler_area_collections_fields_fset'
+  );
+
+
   // Get the list of fields in this view.
   $field_ids = array();
   $defaults = array();
@@ -111,52 +135,40 @@ function tripal_views_handler_area_collections_form($form, $form_state, $view, $
 
     // Add in in any non CSV or Tab formatters to the label.
     $formatters = array();
-    if (tripal_load_include_field_class($field_type)) {
-      $instance_settings = $field_type::$default_instance_settings;
-      if (array_key_exists('download_formatters', $instance_settings)) {
-        foreach ($instance_settings['download_formatters'] as $class_name) {
-          if ($class_name != 'TripalTabDownloader' and $class_name != 'TripalCSVDownloader') {
-            tripal_load_include_downloader_class($class_name);
-              $formatters[] = $class_name::$label;
-          }
-        }
-      }
-    }
-    if (count($formatters) > 0) {
-      $field_label .= ' (' . implode(',' , $formatters) . ')';
+    $field_formatters = tripal_get_field_field_formatters($field, $instance);
+    foreach ($field_formatters as $class_name => $label) {
+      tripal_load_include_downloader_class($class_name);
+      $formatters[] = $class_name::$label;
     }
 
     // Add the field to those supported.
-    $field_ids[$instance['field_id']] =  $field_label;
+    $field_ids[$instance['field_id']]= $field_label;
+
 
     // Automatically check fields that are in the view and not excluded.
+    $checked = FALSE;
     if (array_key_exists($field_name, $view_fields)) {
       if (array_key_exists('exclude', $view_fields[$field_name]) and
           $view_fields[$field_name]['exclude'] == TRUE) {
         continue;
       }
-      $defaults[] = $instance['field_id'];
+      $checked = TRUE;
     }
+
+    $form['save_collection']['fields']['items'] ['select-' . $instance['field_id']] = array(
+      '#type' => 'checkbox',
+      '#title' => $field_label,
+      '#default_value' => $checked,
+    );
+    $form['save_collection']['fields']['items'] ['description-' . $instance['field_id']] = array(
+      '#type' => 'markup',
+      '#markup' => $instance['description']
+    );
+    $form['save_collection']['fields']['items'] ['formatters-' . $instance['field_id']] = array(
+      '#type' => 'markup',
+      '#markup' => join(', ', $formatters)
+    );
   }
-  $form['save_collection']['fields'] = array(
-    '#type' => 'fieldset',
-    '#title' => t('Advanced field selection'),
-    '#description' => t('Please select the fields to include in this data
-      collection. Not all of these fields appear in the search results
-      above but they are available for this content type. By default,
-      tab-delimeted and comma-separated files are genearted for the
-      collection using only the fields selected. However, some fields when
-      selected will generate other downloadable file formats.  Fields that
-      generate other file formats are indicated. '),
-    '#collapsible' => TRUE,
-    '#collapsed' => TRUE,
-  );
-  $form['save_collection']['fields']['field_ids'] = array(
-    '#type' => 'checkboxes',
-    '#title' => t('Field Selection'),
-    '#options' => $field_ids,
-    '#default_value' => $defaults,
-  );
 
   $form['save_collection']['button'] = array(
     '#type' => 'submit',
@@ -174,6 +186,48 @@ function tripal_views_handler_area_collections_form($form, $form_state, $view, $
   $form['#suffix'] = '</div>';
   return $form;
 }
+/**
+ * Theme the fields section of the tripal_views_handler_area_collections form.
+ *
+ * @ingroup tripal_pub
+ */
+function theme_tripal_views_handler_area_collections_fields_fset($variables) {
+  $form = $variables['form'];
+
+  // Organize the elements by the same field id
+  $fields = array();
+  $order = array();
+  $children =  element_children($form);
+  foreach ($children as $key) {
+    list($item, $field_id) = preg_split('/-/', $key);
+    $fields[$field_id][$item] = $form[$key];
+    if (!in_array($field_id, $order)) {
+      $order[] = $field_id;
+    }
+  }
+
+  // Next create a table with each field in a different row.
+  $headers = array('Field', 'Description', 'Supported Files Types');
+  $rows = array();
+  foreach ($order as $field_id) {
+    $rows[] = array(
+      drupal_render($fields[$field_id]['select']),
+      drupal_render($fields[$field_id]['description']),
+      drupal_render($fields[$field_id]['formatters'])
+    );
+  }
+  $table = array(
+    'header' => $headers,
+    'rows' => $rows,
+    'attributes' => array(),
+    'sticky' => TRUE,
+    'caption' => '',
+    'colgroups' => array(),
+    'empty' => '',
+  );
+
+  return theme_table($table);
+}
 
 /**
  *
@@ -196,10 +250,14 @@ function tripal_views_handler_area_collections_form_submit($form, $form_state) {
   $uid = $user->uid;
   $bundle_name = $bundle->name;
 
+  // Get the fields that have been selected
   $selected_fids = array();
-  foreach ($field_ids as $field_id => $is_selected) {
-    if ($is_selected) {
-      $selected_fids[] = $field_id;
+  foreach ($form_state['values'] as $key => $value) {
+    $matches = array();
+    if (preg_match('/select-(\d+)/', $key, $matches)) {
+      if ($value == 1) {
+        $selected_fids[] = $matches[1];
+      }
     }
   }
 
@@ -213,9 +271,9 @@ function tripal_views_handler_area_collections_form_submit($form, $form_state) {
   $collection = tripal_create_collection(array(
     'uid'  => $uid,
     'collection_name' => $collection_name,
+    'description'  => $description,
     'bundle_name' => $bundle_name,
     'ids' => $entities,
     'fields' => $selected_fids,
-    'description'  => $description,
   ));
-}
+}

tripal_chado/api/tripal_chado.custom_tables.api.inc

@@ -126,7 +126,9 @@ function chado_edit_custom_table($table_id, $table_name, $schema, $skip_if_exist
  *
  * @ingroup tripal_custom_tables_api
  */
-function chado_create_custom_table($table, $schema, $skip_if_exists = TRUE, $mview_id = NULL, $redirect = TRUE) {
+function chado_create_custom_table($table, $schema, $skip_if_exists = TRUE,
+    $mview_id = NULL, $redirect = TRUE) {
+
   global $databases;
   $created = 0;
   $recreated = 0;
@@ -204,10 +206,10 @@ function chado_create_custom_table($table, $schema, $skip_if_exists = TRUE, $mvi
   }
   catch (Exception $e) {
     $transaction->rollback();
+    $error = $e->getMessage();
     watchdog_exception('tripal_chado', $e);
-    $error = _drupal_decode_exception($e);
     drupal_set_message(t("Could not add custom table '%table_name': %message.",
-      array('%table_name' => $table, '%message' => $error['!message'])), 'error');
+        array('%table_name' => $table, '%message' => $error)), 'error');
     return FALSE;
   }
 

tripal_chado/includes/TripalFields/ChadoField.inc

@@ -41,12 +41,13 @@ class ChadoField extends TripalField {
     'chado_column' => '',
     // The base table.
     'base_table' => '',
-    // Indicates the download formats for this field.  The list must be the
-    // name of a child class of the TripalFieldDownloader.
-    'download_formatters' => array(
-      'TripalTabDownloader',
-      'TripalCSVDownloader',
-    ),
+  );
+
+  // Indicates the download formats for this field.  The list must be the
+  // name of a child class of the TripalFieldDownloader.
+  public static $download_formatters = array(
+    'TripalTabDownloader',
+    'TripalCSVDownloader',
   );
 
   // The module that manages this field.

tripal_chado/includes/TripalFields/chado_linker__contact/chado_linker__contact.inc

@@ -38,7 +38,6 @@ class chado_linker__contact extends ChadoField {
     'term_fixed' => FALSE,
   );
 
-
   // The default widget for this field.
   public static $default_widget = 'chado_linker__contact_widget';
 

tripal_chado/includes/TripalFields/data__protein_sequence/data__protein_sequence.inc

@@ -35,13 +35,14 @@ class data__protein_sequence extends ChadoField {
     // type. This will create form elements when editing the field instance
     // to allow the site admin to change the term settings above.
     'term_fixed' => FALSE,
-    // Indicates the download formats for this field.  The list must be the
-    // name of a child class of the TripalFieldDownloader.
-    'download_formatters' => array(
-      'TripalTabDownloader',
-      'TripalCSVDownloader',
-      'TripalProteinFASTADownloader',
-    ),
+  );
+
+  // Indicates the download formats for this field.  The list must be the
+  // name of a child class of the TripalFieldDownloader.
+  public static $download_formatters = array(
+    'TripalTabDownloader',
+    'TripalCSVDownloader',
+    'TripalProteinFASTADownloader',
   );
 
   // The default widget for this field.

tripal_chado/includes/TripalFields/data__sequence/data__sequence.inc

@@ -35,13 +35,14 @@ class data__sequence extends ChadoField {
     // type. This will create form elements when editing the field instance
     // to allow the site admin to change the term settings above.
     'term_fixed' => FALSE,
-    // Indicates the download formats for this field.  The list must be the
-    // name of a child class of the TripalFieldDownloader.
-    'download_formatters' => array(
-      'TripalTabDownloader',
-      'TripalCSVDownloader',
-      'TripalNucFASTADownloader',
-    ),
+  );
+
+  // Indicates the download formats for this field.  The list must be the
+  // name of a child class of the TripalFieldDownloader.
+  public static $download_formatters = array(
+    'TripalTabDownloader',
+    'TripalCSVDownloader',
+    'TripalNucFASTADownloader',
   );
 
   // The default widget for this field.

tripal_chado/includes/TripalFields/data__sequence_coordinates/data__sequence_coordinates.inc

@@ -216,6 +216,7 @@ class data__sequence_coordinates extends ChadoField {
     $feature = $entity->chado_record;
     $num_seqs = 0;
 
+    $description = 'schema:description';
     $reference_term = 'data:3002';
     $fmin_term = tripal_get_chado_semweb_term('featureloc', 'fmin');
     $fmax_term = tripal_get_chado_semweb_term('featureloc', 'fmax');
@@ -245,11 +246,14 @@ class data__sequence_coordinates extends ChadoField {
         else {
           $strand = '-';
         }
+        $fmin = $featureloc->fmin + 1;
+        $fmax = $featureloc->fmax;
         $entity->{$field_name}['und'][0] = array(
           'value' => array(
+            $description => $srcfeature . ':' . $fmin . '-' . $fmax . $strand,
             $reference_term => $srcfeature,
-            $fmin_term => $featureloc->fmin + 1,
-            $fmax_term => $featureloc->fmax,
+            $fmin_term => $fmin,
+            $fmax_term => $fmax,
             $strand_term => $strand,
             $phase_term => $featureloc->phase,
           ),

tripal_chado/includes/TripalFields/sbo__relationship/sbo__relationship.inc

@@ -328,9 +328,15 @@ class sbo__relationship extends ChadoField {
     // Example: The genetic_marker, MARKER1 , derives from the sequence_variant, VARIANT1.
     // The above relationship will be shown both on marker and variant pages
     // and as such both subject and object names need to be shown.
-    $entity->{$field_name}['und'][$delta]['value']['SIO:000493'] = 'The ' . $subject_type . ', ' .
-      $subject_name . ', ' . $verb . ' '  . $rel_type_clean . ' the '  .
-      $object_type . ', ' . $object_name . '.';
+    $clause = 'The ' . $subject_type . ', ' .
+        $subject_name . ', ' . $verb . ' '  . $rel_type_clean . ' the '  .
+        $object_type . ', ' . $object_name . '.';
+    $entity->{$field_name}['und'][$delta]['value']['SIO:000493'] = $clause;
+
+    // Adding a label allows us to provide a single text value for the
+    // entire field. It is this text value that can be used in tab/csv
+    // downloaders.
+    $entity->{$field_name}['und'][$delta]['value']['rdfs:label'] = $clause;
 
     $entity->{$field_name}['und'][$delta]['chado-' . $field_table . '__' . $pkey] = $relationship->$pkey;
     $entity->{$field_name}['und'][$delta]['chado-' . $field_table . '__' . $subject_id_key] = $relationship->$subject_id_key->$subject_pkey;

tripal_chado/includes/TripalFields/schema__alternate_name/schema__alternate_name.inc

@@ -79,9 +79,10 @@ class schema__alternate_name extends ChadoField {
     $linker_table = $base_table . '_synonym';
     $options = array('return_array' => 1);
     $record = chado_expand_var($record, 'table', $linker_table, $options);
-    if (count($record->$linker_table) > 0) {
+    $synonyms_linker = $record->$linker_table;
+    if (is_array($synonyms_linker) and count($synonyms_linker) > 0) {
       $i = 0;
-      foreach ($record->$linker_table as $index => $linker) {
+      foreach ($synonyms_linker as $index => $linker) {
         $synonym = $linker->synonym_id;
         $entity->{$field_name}['und'][$i] = array(
           'value' => $synonym->name,

tripal_chado/includes/TripalImporter/GFF3Importer.inc

@@ -1071,13 +1071,13 @@ class GFF3Importer extends TripalImporter {
     if (!$type_id) {
       $result = chado_select_record('feature', array('type_id'), $values);
       if (count($result) > 1) {
-        $this->logMessage("Cannot find feature type for, '%subject' , in 'derives_from' relationship. Multiple matching features exist with this uniquename.",
-            array('%subject' => $object), TRIPAL_WARNING);
+        $this->logMessage("Cannot find feature type for, '!subject' , in 'derives_from' relationship. Multiple matching features exist with this uniquename.",
+            array('!subject' => $object), TRIPAL_WARNING);
         return;
       }
       else if (count($result) == 0) {
-        $this->logMessage("Cannot find feature type for, '%subject' , in 'derives_from' relationship.",
-            array('%subject' => $object), TRIPAL_WARNING);
+        $this->logMessage("Cannot find feature type for, '!subject' , in 'derives_from' relationship.",
+            array('!subject' => $object), TRIPAL_WARNING);
         return '';
       }
       else {

tripal_chado/includes/tripal_chado.cv.inc

@@ -105,13 +105,12 @@ function tripal_cv_cv_edit_form($form, &$form_state) {
     tripal_cv_add_cv_form_fields($form, $form_state, $cv_id);
 
     $form['update'] = array(
-      '#type'         => 'submit',
-      '#value'        => t('Update'),
+      '#type' => 'submit',
+      '#value' => t('Update'),
     );
     $form['delete'] = array(
-      '#type'         => 'submit',
-      '#value'        => t('Delete'),
-      '#attributes'   => array('onclick' => 'if(!confirm("Really Delete?")){return false;}'),
+      '#type' => 'markup',
+      '#markup' => l('delete', 'admin/tripal/loaders/chado_vocabs/chado_cv/delete/' . $cv_id),
     );
   }
   else {
@@ -273,16 +272,6 @@ function tripal_cv_cv_edit_form_submit($form, &$form_state) {
       drupal_set_message(t("Failed to update controlled vocabulary."));
     }
   }
-  if (strcmp($op, 'Delete')==0) {
-    $match = array('cv_id' => $cv_id);
-    $success = chado_delete_record('cv', $match);
-    if ($success) {
-      drupal_set_message(t("Controlled vocabulary deleted"));
-    }
-    else {
-      drupal_set_message(t("Failed to delete controlled vocabulary."));
-    }
-  }
 }
 
 /**
@@ -824,21 +813,38 @@ function tripal_cv_cvtermpath_form_submit($form, &$form_state) {
   }
 }
 
-function tripal_cv_obo_ebi_lookup($accession) {
-  print_r("accession: $accession\n");
-  //The IRI of the terms, this value must be double URL encoded
-  $iri = urlencode("http://purl.obolibrary.org/obo/" . $accession);
-  //Grab just the ontology from the $accession.
-  $parts = explode(':', $accession);
-  $ontology = $parts[0];
-  print_r("ontology: $ontology\n");
-  $options = array();
-  $full_url = 'http://www.ebi.ac.uk/ols/api/ontologies/' . $ontology . '/' . 'terms/' . $iri;
-  $response = drupal_http_request($full_url, $options);
-  if(!empty($response)){
-    $response = drupal_json_decode($response->data);
-  }
-  print_r($response);
-  return $response;
-  // php-eval "module_load_include('inc', 'tripal_chado', 'sites/all/modules/tripal/tripal_chado/includes/tripal_chado.cv'); tripal_cv_obo_ebi_lookup('BFO:0000050')"
+/**
+ * A confirmation form for deleting a controlled vocabulary.
+ */
+function tripal_cv_cv_delete_form($form, &$form_state, $cv_id) {
+
+  $cv = tripal_get_cv(array('cv_id' => $cv_id));
+
+  $form['cv_id'] = array(
+    '#type' => 'value',
+    '#value' => $cv_id,
+  );
+
+  return confirm_form($form,
+    t('Confirm removal of the vocabulary: "' . $cv->name . '"? '),
+    'admin/tripal/loaders/chado_vocabs/chado_cv/edit/' . $cv_id,
+    t('WARNING: removal of a vocabulary will result in removal of all terms, and any associations used for other records in the site that use those terms.')
+  );
+}
+
+/**
+ * Implements the submit hook for tripal_cv_cv_delete_form.
+ */
+function tripal_cv_cv_delete_form_submit($form, &$form_state) {
+  $cv_id = $form_state['values']['cv_id'];
+
+  try {
+    $match = array('cv_id' => $cv_id);
+    $success = chado_delete_record('cv', $match);
+    drupal_set_message(t("Controlled vocabulary deleted"));
+    drupal_goto('admin/tripal/loaders/chado_vocabs/chado_cv');
+  }
+  catch (Exception $e) {
+    drupal_set_message(t("Failed to delete controlled vocabulary."), 'error');
+  }
 }

tripal_chado/includes/tripal_chado.field_storage.inc

@@ -601,6 +601,56 @@ function tripal_chado_field_storage_query($query) {
     $cquery->condition('TE.' . $condition['column'], $condition['value']);
   }
 
+  // If there is an entity_id filter then apply that.
+  if (array_key_exists('entity_id', $query->entityConditions)) {
+    $value = $query->entityConditions['entity_id']['value'];
+    $op = '';
+    if (array_key_exists('op', $query->entityConditions['entity_id'])) {
+      $op = $query->entityConditions['entity_id']['op'];
+    }
+    // Handle a single entity_id
+    if (!is_array($value)) {
+      switch ($op) {
+        case 'CONTAINS':
+          $op = 'LIKE';
+          $value = '%' . $value . '%';
+          break;
+        case 'STARTS_WITH':
+          $op = 'LIKE';
+          $value = $value . '%';
+          break;
+        case 'BETWEEN':
+          // This case doesn't make sense for a single value. So, just set it
+          // equal to the end.
+          $op = '=';
+        default:
+          $op = $op ? $op : '=';
+      }
+    }
+    // Handle multiple entitie IDs.
+    else {
+      switch ($op) {
+        case 'CONTAINS':
+          $op = 'IN';
+          break;
+        case 'STARTS_WITH':
+          $op = 'IN';
+          break;
+        case 'BETWEEN':
+          $op = 'BETWEEN';
+        default:
+          $op = 'IN';
+      }
+    }
+    if ($op == 'BETWEEN') {
+      $cquery->condition('TE.id', $value[0], '>');
+      $cquery->condition('TE.id', $value[1], '<');
+    }
+    else {
+      $cquery->condition('TE.id', $value, $op);
+    }
+  }
+
   // Now set any ordering.
   foreach ($query->order as $index => $sort) {
     // Add in property ordering.

tripal_chado/includes/tripal_chado.fields.inc

@@ -1337,7 +1337,7 @@ function tripal_chado_bundle_instances_info_custom(&$info, $entity_type, $bundle
       'entity_type' => $entity_type,
       'bundle' => $bundle->name,
       'label' => 'Sequence',
-      'description' => 'All available sequences for this record.',
+      'description' => 'One or more molecular sequences, possibly with associated annotation.',
       'required' => FALSE,
       'settings' => array(
         'auto_attach' => FALSE,

tripal_chado/includes/tripal_chado.migrate.inc

@@ -927,7 +927,7 @@ function tripal_chado_migrate_organism_image_add_file ($fid, $entity_id, $bundle
     file_usage_add($file, 'file', 'TripalEntity', $entity_id);
     $image_file = (array) $file;
     // Attached it to the entity
-    /* 
+    /*
     $entities = entity_load('TripalEntity', array($entity_id));
     $entity = $entities[$entity_id];
     $image = array(

tripal_chado/tripal_chado.module

@@ -607,6 +607,17 @@ function tripal_chado_menu() {
     'file path' => drupal_get_path('module', 'tripal_chado'),
     'type' => MENU_CALLBACK,
   );
+  $items['admin/tripal/loaders/chado_vocabs/chado_cv/delete/%'] = array(
+    'title' => 'Delete a Controlled Vocabulary',
+    'description' => 'Delete a controlled vocabulary.',
+    'page callback' => 'drupal_get_form',
+    'page arguments' => array('tripal_cv_cv_delete_form', 6),
+    'access callback' => 'user_access',
+    'access arguments' => array('administer controlled vocabularies'),
+    'file' => 'includes/tripal_chado.cv.inc',
+    'file path' => drupal_get_path('module', 'tripal_chado'),
+    'type' => MENU_CALLBACK,
+  );
   $items['admin/tripal/loaders/chado_vocabs/chado_cv/add'] = array(
     'title' => 'Add a Controlled Vocabulary',
     'description' => 'Manually a new controlled vocabulary.',
@@ -1024,22 +1035,40 @@ function tripal_chado_node_delete($node) {
  * backend indicates they are not appropriate.
  */
 function tripal_chado_form_field_ui_field_edit_form_alter(&$form, &$form_state, $form_id) {
-  // For entity fields added by Tripal Entities we don't want the
-  // the end-user to change the cardinality and the required fields
-  // such that record can't be saved in Chado.
 
-  // TODO: we shouldn't check for specific field types here
-  // (e.g. chado_linker_prop). That should be done via the TripalField
-  // functions.
   if ($form['#instance']['entity_type'] == 'TripalEntity') {
+
+    // For entity fields added by Tripal Entities we don't want the
+    // the end-user to change the cardinality and the required fields
+    // such that record can't be saved in Chado.
+    // TODO: we shouldn't check for specific field types here
+    // (e.g. chado_linker_prop). That should be done via the TripalField
+    // functions.
     if ($form['#field']['storage']['type'] == 'field_chado_storage' and
         $form['#field']['type'] != 'chado_linker__prop') {
       $form['field']['cardinality']['#access'] = FALSE;
       $form['instance']['required']['#access'] = FALSE;
     }
+
+    // If this is a Chado field we need to preserve the Chado elements of the
+    // settings or they will be lost if a user edits the field settings.
+    if (array_key_Exists('chado_table', $form['#instance']['settings'])) {
+      $form['instance']['settings']['base_table'] = array(
+        '#type' => 'value',
+        '#value' => $form['#instance']['settings']['base_table'],
+      );
+      $form['instance']['settings']['chado_table'] = array(
+        '#type' => 'value',
+        '#value' => $form['#instance']['settings']['chado_table'],
+      );
+      $form['instance']['settings']['chado_column'] = array(
+        '#type' => 'value',
+        '#value' => $form['#instance']['settings']['chado_column'],
+      );
+    }
   }
 
-  // TODO: don't the the maximum length be larger than the field size.
+  // TODO: don't let the maximum length be larger than the field size.
 }
 
 

tripal_chado_views/tripal_chado_views.module

@@ -74,7 +74,7 @@ function tripal_chado_views_menu() {
   $items['admin/tripal/storage/chado/views-integration/edit/%'] = array(
     'title' => 'Edit Views Integration',
     'page callback' => 'drupal_get_form',
-    'page arguments' => array('tripal_chado_views_integration_form', 4),
+    'page arguments' => array('tripal_chado_views_integration_form', 6),
     'access arguments' => array('manage tripal_views_integration'),
     'type' => MENU_CALLBACK,
   );

tripal_ws/api/tripal_ws.api.inc

@@ -102,4 +102,776 @@ function tripal_load_include_web_service_class($class) {
     }
   }
   return FALSE;
+}
+
+/**
+ * Adds a new site to the web services table.
+ *
+ * @param $name
+ *   Name of site to be included.
+ * @param $url
+ *   URL of site to be added.
+ * @param $version
+ *   Version of the API being used. default to 1
+ * @param $description
+ *    A description of the site and any additional info that
+ *    would be helpful for admins.
+ *
+ * @return
+ *   TRUE if the site is successfully added, FALSE otherwise.
+ */
+function tripal_add_site($name, $url, $version, $description) {
+  $check_url = NULL;
+  $check_name = NULL;
+  $write_to_db = TRUE;
+  // When inserting a record.
+  $check_url =
+    db_select('tripal_sites', 'ts')
+      ->fields('ts', array('id'))
+      ->condition('url', $url)
+      ->condition('version', $version)
+      ->execute()
+      ->fetchField();
+
+  $check_name =
+    db_select('tripal_sites', 'ts')
+      ->fields('ts', array('id'))
+      ->condition('name', $name)
+      ->execute()
+      ->fetchField();
+
+  if ($check_url) {
+    drupal_set_message(t('The URL and version is used by another site.'), 'error');
+    $write_to_db = FALSE;
+  }
+
+  if ($check_name) {
+    drupal_set_message(t('The name is used by another site.'), 'error');
+    $write_to_db = FALSE;
+  }
+  if ($write_to_db === TRUE) {
+    db_insert('tripal_sites')
+    ->fields(array(
+        'name' => $name,
+        'url' => $url,
+        'version' => $version,
+        'description' => $description
+      ))
+      ->execute();
+    drupal_set_message(t('Tripal site \'' . $name . '\' has been added.'));
+    return $write_to_db;
+  }
+
+  return $write_to_db;
+}
+
+/**
+ * Remove a site from the web services table.
+ *
+ * @param $record_id
+ *   ID of the record to be deleted.
+ *
+ * @return
+ *   TRUE if the record was successfully deleted, FALSE otherwise.
+ */
+function tripal_remove_site($record_id) {
+  if ($record_id) {
+    db_delete('tripal_sites')
+      ->condition('id', $record_id)
+      ->execute();
+    drupal_set_message('The Tripal site \'' . $record_id . '\' has been removed.');
+    return TRUE;
+  }
+  return FALSE;
+}
+
+/**
+ * Makes a request to the "content" service of a remote Tripal web site.
+ *
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @param $path
+ *   The web service path for the content (excluding
+ *   'web-servcies/vX.x/content').  To retrieve the full content listing
+ *   leave this paramter empty.
+ * @param $query
+ *   An query string to appear after the ? in a URL.
+ *
+ * @return
+ *   The JSON response formatted in a PHP array or FALSE if a problem occured.
+ */
+function tripal_get_remote_content($site_id, $path = '', $query = '') {
+
+  if (!$site_id) {
+    throw new Exception('Please provide a numeric site ID for the tripal_get_remote_content function.');
+  }
+
+  // Fetch the record for this site id.
+  $remote_site = db_select('tripal_sites', 'ts')
+    ->fields('ts')
+    ->condition('ts.id', $site_id)
+    ->execute()
+    ->fetchObject();
+
+  if (!$remote_site) {
+    tripal_report_error('tripal_ws', TRIPAL_ERROR,
+      t('Could not find a remote tripal site using the id provided: !id.',
+        array('!id' => $site_id)));
+    return FALSE;
+  }
+
+  // Build the URL to the remote web services.
+  $ws_version = $remote_site->version;
+  $ws_url = $remote_site->url;
+  $ws_url = trim($ws_url, '/');
+  $ws_url .= '/web-services/content/' . $ws_version . '/' . $path;
+
+  // Build the Query and make and substitions needed.
+  if ($query) {
+    $ws_url = $ws_url . '?' . $query;
+  }
+
+  // TODO: something is wrong here, the query is not being recognized on
+  // the remote Tripal site. It's just returning the default.
+  $data = drupal_http_request($ws_url);
+
+  if (!$data) {
+    tripal_report_error('tripal_ws', TRIPAL_ERROR,
+        t('Could not connect to the remote web service.'));
+    return FALSE;
+  }
+
+  // If the data object has an error then this is some sort of
+  // connection error (not a Tripal web servcies error).
+  if (property_exists($data, 'error')) {
+    tripal_report_error('tripal_ws', TRIPAL_ERROR,
+        'Remote web Services reports the following error: !error. Using URL: !url',
+        array('!error' => $error, '!url' => $ws_url));
+    return FALSE;
+  }
+
+  // We got a response, so convert it to a PHP array.
+  $data = drupal_json_decode($data->data);
+
+  // Check if there was a Tripal Web Services error.
+  if (array_key_exists('error', $data)) {
+    $error = '</pre>' . print_r($data['error'], TRUE) . '</pre>';
+    tripal_report_error('tripal_ws', TRIPAL_ERROR,
+        'Tripal remote web services reports the following error: !error. Using URL: !url',
+        array('!error' => $error, '!url' => $ws_url));
+    return FALSE;
+  }
+
+  return $data;
+}
+
+/**
+ * Retrieves the JSON-LD context for any remote Tripal web service.
+ *
+ * @param $context_url
+ *   The Full URL for the context file on the remote Tripal site. This URL
+ *   can be found in the '@context' key of any response from a remote Tripal
+ *   web services call.
+ * @param $cache_id
+ *   A unique ID for caching of this context result to speed furture
+ *   queries.
+ * @return
+ *   The JSON-LD context mapping array, or FALSE if the context could not
+ *   be retrieved.
+ */
+function tripal_get_remote_context($context_url, $cache_id) {
+
+  if (!$context_url) {
+    throw new Exception('PLease provide a context_url for the tripal_get_remote_context function.');
+  }
+  if (!$cache_id) {
+    throw new Exception('PLease provide unique $cache_id for the tripal_get_remote_context function.');
+  }
+
+  if ($cache = cache_get($cache_id)) {
+    return $cache->data;
+  }
+
+  $context = drupal_http_request($context_url);
+  if (!$context) {
+    tripal_report_error('tripal_ws', TRIPAL_ERROR,
+        'There was a poblem retrieving the context from the remote site: !context_url.',
+        array('!context_url' => $context_url));
+    return FALSE;
+  }
+  $context = drupal_json_decode($context->data);
+  $context = $context['@context'];
+  cache_set($cache_id, $context);
+  return $context;
+}
+
+/**
+ * Retrieves the JSON-LD context for a bundle or field on a remote Tripal site.
+ *
+ * The $site_id, $bundle_accession and $field_accession variables are not
+ * needed to retrieve the context, but are used for caching the context to
+ * make subsequent calls execute faster.  This function is meant to be used
+ * only for the 'content' service provided by Tripal.
+ *
+ * @param $site_id
+ *    The numeric site ID for the remote Tripal site.
+ * @param $context_url
+ *   The Full URL for the context file on the remote Tripal site. This URL
+ *   can be found in the '@context' key of any response from a remote Tripal
+ *   web services call.
+ * @param $bundle_accession
+ *   The controlled vocabulary term accession for the content type
+ *   on the remote Tripal site.
+ * @param $field_accession
+ *   The controlled vocabulary term accession for the property (i.e. field) of
+ *   the Class (i.e. content type).
+ * @return
+ *   The JSON-LD context mapping array.
+ */
+function tripal_get_remote_content_context($site_id, $context_url, $bundle_accession, $field_accession = '') {
+  $cache_id = substr('trp_ws_context_' . $site_id . '-' . $bundle_accession . '-' . $field_accession, 0, 254);
+  $context = tripal_get_remote_context($context_url, $cache_id);
+  return $context;
+}
+
+/**
+ * Clears the cached remote site documentation and context.
+ *
+ * When querying a remote website, the site's API documenation and page context
+ * is cached to make re-use of that information easier in the future. This
+ * function can be used to clear those caches.
+ *
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ */
+function tripal_clear_remote_cache($site_id) {
+  if (!$site_id) {
+    throw new Exception('Please provide a numeric site ID for the tripal_clear_remote_cache function.');
+  }
+  cache_clear_all('trp_ws_context_' . $site_id , 'cache', TRUE);
+  cache_clear_all('trp_ws_doc_' . $site_id , 'cache', TRUE);
+}
+/**
+ * Retrieves the API documentation for a remote Tripal web service.
+ *
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @return
+ *   The vocabulary of a remote Tripal web service, or FALSE if an error
+ *   occured.
+ */
+function tripal_get_remote_API_doc($site_id) {
+  $site_doc = '';
+
+  if (!$site_id) {
+    throw new Exception('Please provide a numeric site ID for the tripal_get_remote_API_doc function.');
+  }
+
+  $cache_name = 'trp_ws_doc_' . $site_id;
+  if ($cache = cache_get($cache_name)) {
+    return $cache->data;
+  }
+
+  // Get the site url from the tripal_sites table.
+  $remote_site = db_select('tripal_sites', 'ts')
+    ->fields('ts')
+    ->condition('ts.id', $site_id)
+    ->execute()
+    ->fetchObject();
+
+  if (!$remote_site) {
+    throw new Exception(t('Cannot find a remote site with id: "!id"', array('!id' => $site_id)));
+  }
+
+  // Build the URL to the remote web services.
+  $ws_version = $remote_site->version;
+  $ws_url = $remote_site->url;
+  $ws_url = trim($ws_url, '/');
+  $ws_url .= '/web-services/doc/' . $ws_version;
+
+  // Build and make the request.
+  $options = [];
+  $data = drupal_http_request($ws_url, $options);
+
+  if (!$data) {
+    tripal_report_error('tripal_ws', TRIPAL_ERROR,
+        t('Could not connect to the remote web service.'));
+    return FALSE;
+  }
+
+  // If the data object has an error then this is some sort of
+  // connection error (not a Tripal web servcies error).
+  if (property_exists($data, 'error')) {
+    tripal_report_error('tripal_ws', TRIPAL_ERROR,
+        'Remote web services document reports the following error: !error. Using URL: !url',
+      array('!error' => $error, '!url' => $ws_url));
+    return FALSE;
+  }
+
+  // We got a response, so convert it to a PHP array.
+  $site_doc = drupal_json_decode($data->data);
+
+  // Check if there was a Tripal Web Services error.
+  if (array_key_exists('error', $data)) {
+    $error = '</pre>' . print_r($data['error'], TRUE) . '</pre>';
+    tripal_report_error('tripal_ws', TRIPAL_ERROR,
+        'Tripal Remote web services document reports the following error: !error. Using URL: !url',
+      array('!error' => $error, '!url' => $ws_url));
+    return FALSE;
+  }
+
+  cache_set($cache_name, $site_doc);
+
+  return $site_doc;
+}
+
+/**
+ * Queries a remote site for an array of bulk entity ids.
+ *
+ * This function returns an array of "fake" entities containing values
+ * for fields specified.
+ *
+ * @param $remote_entity_ids
+ *   Array of the remote ids.
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @param $bundle_accession
+ *   The controlled vocabulary term accession for the content type
+ *   on the remote Tripal site.
+ * @param $field_ids
+ *   The controlled vocabulary term accessions for the fields available
+ *   on the remote content type.  Any remote fields that matches these IDs will
+ *   be added to the entity returned.
+ * @return
+ *    An array of fake entity objects where the key is the entity_id and
+ *    the value is the object.
+ */
+function tripal_load_remote_entities($remote_entity_ids, $site_id, $bundle_accession, $field_ids) {
+
+  if (!$remote_entity_ids) {
+    throw new Exception('Please provide the list of remote entity ids for the tripal_load_remote_entities function.');
+  }
+  if (!is_array($remote_entity_ids)) {
+    throw new Exception('Please provide an array for the remote entity ids for the tripal_load_remote_entities function.');
+  }
+  if (!$site_id) {
+    throw new Exception('Please provide a numeric site ID for the tripal_load_remote_entities function.');
+  }
+  if (!$bundle_accession) {
+    throw new Exception('Please provide the bundle accession for the tripal_load_remote_entities function.');
+  }
+  if (!$field_ids) {
+    throw new Exception('Please provide the list of field IDs for the tripal_load_remote_entities function.');
+  }
+  if (!is_array($field_ids)) {
+    throw new Exception('Please provide an array for the field IDs for the tripal_load_remote_entities function.');
+  }
+
+  // Get the site documentation (loads from cache if already retrieved).
+  $site_doc = tripal_get_remote_API_doc($site_id);
+
+  // Generate an array for the query and then execute it.
+  $query = 'page=1&limit=' . count($remote_entity_ids) .
+    '&ids=' . urlencode(implode(",", $remote_entity_ids)) .
+    '&fields=' . urlencode(implode(",", $field_ids));
+
+  $results = tripal_get_remote_content($site_id, $bundle_accession, $query);
+  if (!$results) {
+    return FALSE;
+  }
+
+  // Get the context JSON for this remote entity, we'll use it to map
+  $context_url = $results['@context'];
+  $context = tripal_get_remote_content_context($site_id, $context_url, $bundle_accession);
+  if (!$context) {
+    return $entity;
+  }
+
+  $total_items = $results['totalItems'];
+  $members = $results['member'];
+
+  $entities = array();
+  foreach($members as $member) {
+    // Start building the fake entity.
+    $entity_id = preg_replace('/^.*?(\d+)$/', '$1', $member['@id']);
+    $entity = new stdClass();
+    $entity->entityType = 'TripalEntity';
+    $entity->entityInfo = [];
+    $entity->id = $entity_id;
+    $entity->type = 'TripalEntity';
+    $entity->bundle = $bundle_accession;
+    $entity->site_id = $site_id;
+
+    $member = _tripal_update_remote_entity_field($member, $context, 1);
+    foreach ($member as $field_id => $value) {
+      $field = tripal_get_remote_field_info($site_id, $bundle_accession, $field_id);
+      $instance = tripal_get_remote_field_instance_info($site_id, $bundle_accession, $field_id);
+      $field_name = $field['field_name'];
+      $entity->{$field_name}['und'][0]['value'] = $value;
+    }
+
+    $entities[$entity_id] = $entity;
+  }
+  return $entities;
+}
+
+/**
+ * Queries a remote site for an entity.
+ *
+ * This function returns a "fake" entity containing values for all fields
+ * specified.
+ *
+ * @param $remote_entity_id
+ *   A remote entity ID.
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @param $bundle_accession
+ *   The controlled vocabulary term accession for the content type
+ *   on the remote Tripal site.
+ * @param $field_ids
+ *   The controlled vocabulary term accessions for the fields available
+ *   on the remote content type.  Any remote fields that matches these IDs will
+ *   be added to the entity returned.
+ * @return
+ *    A fake entity object.
+ */
+function tripal_load_remote_entity($remote_entity_id, $site_id, $bundle_accession, $field_ids) {
+
+  // Get the site documentation (loads from cache if already retrieved).
+  $site_doc = tripal_get_remote_API_doc($site_id);
+
+  // Get the remote entity and create the fake entity.
+  $remote_entity = tripal_get_remote_content($site_id, $bundle_accession . '/' . $remote_entity_id);
+  if (!$remote_entity) {
+    return FALSE;
+  }
+
+  // Start building the fake entity.
+  $entity = new stdClass();
+  $entity->entityType = 'TripalEntity';
+  $entity->entityInfo = [];
+  $entity->id = $remote_entity_id;
+  $entity->type = 'TripalEntity';
+  $entity->bundle = $bundle_accession;
+  $entity->site_id = $site_id;
+
+  // Get the context JSON for this remote entity, we'll use it to map
+  $context_url = $remote_entity['@context'];
+  $context = tripal_get_remote_content_context($site_id, $context_url, $bundle_accession);
+  if (!$context) {
+    return $entity;
+  }
+
+  // Iterate through the fields and the those values to the entity.
+  foreach ($field_ids as $field_id) {
+
+    $field = tripal_get_remote_field_info($site_id, $bundle_accession, $field_id);
+    $instance = tripal_get_remote_field_instance_info($site_id, $bundle_accession, $field_id);
+    $field_name = $field['field_name'];
+
+    $field_key = '';
+    foreach ($context as $k => $v) {
+      if (!is_array($v) and $v == $field_id) {
+        $field_key = $k;
+      }
+    }
+
+    // If the field is not in this remote bundle then add an empty value.
+    if (!$field_key) {
+      $entity->{$field_name}['und'][0]['value'] = '';
+      continue;
+    }
+    if (!array_key_exists($field_key, $remote_entity)) {
+      $entity->{$field_name}['und'][0]['value'] = '';
+      continue;
+    }
+
+    // If the key is for a field that is not "auto attached' then we need
+    // to get that field through a separate call.
+    $attached = TRUE;
+    if (array_key_exists($field_id, $context) and is_array($context[$field_id]) and
+        array_key_exists('@type', $context[$field_id]) and $context[$field_id]['@type'] == '@id'){
+          $attached = FALSE;
+    }
+
+    // Set the value for this field.
+    $value = '';
+    if (is_array($remote_entity[$field_key])) {
+      $value = _tripal_update_remote_entity_field($remote_entity[$field_key], $context, 1);
+    }
+    else {
+      $value = $remote_entity[$field_key];
+    }
+
+    // If the field is not attached then we have to query another level.
+    if (!$attached) {
+
+      $field_data = drupal_http_request($value);
+      if (!$field_data) {
+        tripal_report_error('tripal_ws', TRIPAL_ERROR,
+           'There was a poblem retrieving the unattached field, "!field:", for the remote entity: !entity_id.',
+            array('!field' => $field_id, '!entity_id' => $remote_entity_id));
+        $value = '';
+      }
+      $field_data = drupal_json_decode($field_data->data);
+
+      // Get the context for this field so we can map the keys to the
+      // controlled vocabulary accessions. If it fails then skip this field.
+      $field_context_url = $field_data['@context'];
+      $field_context = tripal_get_remote_content_context($site_id, $field_context_url, $bundle_accession, $field_id);
+      if (!$field_context) {
+        continue;
+      }
+      $value = _tripal_update_remote_entity_field($field_data, $field_context);
+    }
+    $entity->{$field_name}['und'][0]['value'] = $value;
+  }
+  return $entity;
+}
+
+/**
+ * A helper function for the tripal_get_remote_entity() function.
+ *
+ * This function converts the field's key elements to their
+ * vocabulary term accessions.
+ *
+ * @param $field_data
+ *   The field array as returned by web services.
+ * @param $context
+ *   The web service JSON-LD context for the bundle to which the field belongs.
+ */
+function _tripal_update_remote_entity_field($field_data, $context, $depth = 0) {
+
+
+    // Check if this is an array.
+    if ($field_data['@type'] == 'Collection') {
+      $members = array();
+      foreach ($field_data['member'] as $member) {
+        $next_depth = $depth + 1;
+        $members[] = _tripal_update_remote_entity_field($member, $context, $next_depth);
+      }
+
+      // If we only have one item then just return it as a single item.
+      // TODO: we may need to check cardinality of the field and be more
+      // strict about how we return the value.
+      if ($field_data['totalItems'] == 1){
+        return $members[0];
+      }
+      else {
+        return $members;
+      }
+    }
+
+    $value = array();
+    foreach ($field_data as $k => $v) {
+      // Skip the JSON-LD keys.
+      if ($k == '@id' or $k == '@type' or $k == '@context') {
+        continue;
+      }
+      // Find the term accession for this element, and if the key's value is an
+      // array then recurse.
+      $accession = $context[$k];
+      if (is_array($v)) {
+        $next_depth = $depth + 1;
+        $subvalue = _tripal_update_remote_entity_field($v, $context, $next_depth);
+        $value[$accession] = $subvalue;
+      }
+      else {
+        $value[$accession] = $v;
+      }
+    }
+    return $value;
+
+}
+
+/**
+ * Behaves similar to the field_info_field() function but for remote fields.
+ *
+ * Returns a "fake" field info array for fields attached to content types
+ * on remote Tripal sites.
+ *
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @param $bundle_accession
+ *   The controlled vocabulary term accession for the content type
+ *   on the remote Tripal site.
+ * @param $field_accession
+ *   The controlled vocabulary term accession for the property (i.e. field) of
+ *   the Class (i.e. content type).
+ * @return
+ *   An array similar to that returned by the field_info_field function
+ *   of Drupal for local fields.
+ */
+function tripal_get_remote_field_info($site_id, $bundle_accession, $field_accession){
+
+  // Get the site documentation (loads from cache if already retrieved).
+  $site_doc = tripal_get_remote_API_doc($site_id);
+
+  // Get the property from the document for this field.
+  $property = tripal_get_remote_field_doc($site_id, $bundle_accession, $field_accession);
+
+  // Now create the fake field and instance.
+  list($vocab, $accession) = explode(':', $field_accession);
+  $field_name = 'tripal_remote_site_' . $site_id . '_' . $field_accession;
+
+  $field = array(
+    'field_name' => $field_name,
+    'type' => $field_name,
+    'storage' => array(
+      'type' => 'tripal_remote_site'
+    ),
+  );
+  return $field;
+}
+
+/**
+ * Behaves similar to the field_info_instance() function but for remote fields.
+ *
+ * Returns a "fake" instance info array for fields attached to content types
+ * on remote Tripal sites.
+ *
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @param $bundle_accession
+ *   The controlled vocabulary term accession for the content type
+ *   on the remote Tripal site.
+ * @param $field_accession
+ *   The controlled vocabulary term accession for the property (i.e. field) of
+ *   the Class (i.e. content type).
+ * @return
+ *   An array similar to that returned by the field_info_instance function
+ *   of Drupal for local fields.
+ */
+function tripal_get_remote_field_instance_info($site_id, $bundle_accession, $field_accession){
+
+  // Get the site documentation (loads from cache if already retrieved).
+  $site_doc = tripal_get_remote_API_doc($site_id);
+
+  // Get the property from the document for this field.
+  $property = tripal_get_remote_field_doc($site_id, $bundle_accession, $field_accession);
+
+  list($vocab, $accession) = explode(':', $field_accession);
+  $field_name = 'tripal_remote_site_' . $site_id . '_' . $field_accession;
+
+  list($vocab, $accession) = explode(':', $field_accession);
+  $instance = array(
+    'label' => $property['hydra:title'],
+    'description' => $property['hydra:description'],
+    'formatters' => $property['tripal_formatters'],
+    'settings' => array(
+      'term_vocabulary' => $vocab,
+      'term_accession' => $accession
+    ),
+    'field_name' => $field_name,
+    'entity_type' => 'TripalEntity',
+    'bundle_name' => $bundle_accession,
+  );
+  return $instance;
+}
+
+
+/**
+ * Retreive the content type information from a remote Tripal site.
+ *
+ * The array returned is equivalent to the Hydra Vocabulary "supportedClass"
+ * stanza.
+ *
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @param $bundle_accession
+ *   The controlled vocabulary term accession for the content type
+ *   on the remote Tripal site.
+ * @return
+ *   A PHP array corresponding to the Hydra Class stanza (i.e. a content type).
+ *   Returns NULL if the class ID cannot be found.
+ */
+function tripal_get_remote_content_doc($site_id, $bundle_accession){
+
+  // Get the site documentation (loads from cache if already retrieved).
+  $site_doc = tripal_get_remote_API_doc($site_id);
+
+  // Get the class that matches this bundle.
+  $classes = $site_doc['supportedClass'];
+  $class = NULL;
+  foreach ($classes as $item) {
+    if ($item['@id'] == $bundle_accession) {
+      return $item;
+    }
+  }
+  return NULL;
+}
+
+/**
+ * Retrieves the field information for a content type from a remote Tripal site.
+ *
+ * The array returned is equivalent to the Hydra Vocabulary "supportedProperty"
+ * stanza that belongs to a Hydra Class (content type).
+ *
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @param $bundle_accession
+ *   The controlled vocabulary term accession for the content type
+ *   on the remote Tripal site.
+ * @param $field_accession
+ *   The controlled vocabulary term accession for the property (i.e. field) of
+ *   the Class (i.e. content type).
+ * @return
+ *   A PHP array corresponding to the Hydra property stanza (field) that
+ *   belongs to the given Class (i.e. a content type).  Retruns NULL if the
+ *   property cannot be found.
+ */
+function tripal_get_remote_field_doc($site_id, $bundle_accession, $field_accession){
+
+  // Get the site documentation (loads from cache if already retrieved).
+  $site_doc = tripal_get_remote_API_doc($site_id);
+
+  $class = tripal_get_remote_content_doc($site_id, $bundle_accession);
+  $properties = $class['supportedProperty'];
+  foreach ($properties as $item) {
+    if ($item['property'] == $field_accession) {
+      return $item;
+    }
+  }
+  return NULL;
+}
+
+
+/**
+ * Retrieves the list of download formatters for a remote field.
+ *
+ * All Tripal fields support the abilty for inclusion in files that can
+ * downloaded.  This function is used to identify what formatters these
+ * fields are appropriate for. If those download formatter classes exist
+ * on this site then the field can be used with that formatter.
+ *
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @param $bundle_accession
+ *   The controlled vocabulary term accession for the content type
+ *   on the remote Tripal site.
+ * @param $field_accession
+ *   The controlled vocabulary term accession for the property (i.e. field) of
+ *   the Class (i.e. content type).
+ * @return
+ *   An array of field downloader class names that are compoatible with the
+ *   field and which exist on this site.
+ */
+function tripal_get_remote_field_formatters($site_id, $bundle_accession, $field_accession) {
+
+  $flist = array();
+
+  // Get the site documentation (loads from cache if already retrieved).
+  $site_doc = tripal_get_remote_API_doc($site_id);
+  $property = tripal_get_remote_field_doc($site_id, $bundle_accession, $field_accession);
+  if (!$property) {
+    return $flist;
+  }
+
+  $formatters = $property['tripal_formatters'];
+  foreach($formatters as $formatter) {
+    if (tripal_load_include_downloader_class($formatter)) {
+      $flist[$formatter] = $formatter::$full_label;
+    }
+  }
+  return $flist;
 }

tripal_ws/includes/TripalFields/remote__data/remote__data.inc

@@ -205,45 +205,7 @@ class remote__data extends WebServicesField {
        list($ctype, $qdata) = explode('?', $query);
      }
 
-     // Build the URL to the remote web services.
-     $ws_version = $this->remote_site->version;
-     $ws_url = $this->remote_site->url;
-     $ws_url = trim($ws_url, '/');
-     $ws_url .= '/web-services/content/' . $ws_version . '/' . $ctype;
-
-     // Build the Query and make and substitions needed.
-     //dpm($ws_url . '?' . $query);
-     $options = array(
-       'data' => $qdata,
-     );
-     $data = drupal_http_request($ws_url, $options);
-
-     if (!$data) {
-       tripal_report_error('tripal_ws', TRIPAL_ERROR,
-           t('Could not connect to the remote web service.'));
-       return FALSE;
-     }
-
-     // If the data object has an error then this is some sort of
-     // connection error (not a Tripal web servcies error).
-     if (property_exists($data, 'error')) {
-       tripal_report_error('tripal_ws', TRIPAL_ERROR,
-           'Web Services error on remote site: %error.',
-           array('%error' => $data->error));
-       return FALSE;
-     }
-
-     // We got a response, so convert it to a PHP array.
-     $data = drupal_json_decode($data->data);
-
-     // Check if there was a Tripal Web Services error.
-     if (array_key_exists('error', $data)) {
-       $error = '</pre>' . print_r($data['error'], TRUE) . '</pre>';
-       tripal_report_error('tripal_ws', TRIPAL_ERROR,
-           'Web Services error on remote site: %error.',
-           array('%error' => $error));
-       return FALSE;
-     }
+     $data = tripal_get_remote_content($this->remote_site->site_id, $query);
 
      return $data;
    }

tripal_ws/includes/TripalWebService.inc

@@ -83,7 +83,7 @@ class TripalWebService {
 
     $this->addDocClass(array(
       "id" => "http://www.w3.org/ns/hydra/core#Resource",
-      "name" => 'resource',
+      "term" => 'hydra:Resource',
       "title" => "Resource",
       'subClassOf' => NULL,
     ));
@@ -345,9 +345,7 @@ class TripalWebService {
    *    An array of key/value pairs providing the details for the class. Valid
    *    keys include:
    *      - id: The unique IRI for this class.
-   *      - name: a computer-readable name for this class (i.e. no spaces,
-   *        no special characters).  This name is used to construct
-   *        type identifiers for operations.
+   *      - term: the accession for the term for this class.
    *      - title:  The title for the resource that this Class represents.
    *      - description: (Optional). A description of the resource.
    *      - subClassOf: (Optional). If this class is a subclass of another
@@ -416,7 +414,11 @@ class TripalWebService {
 
     // Set the Class ID.
     $class_id = $details['id'];
-    $supported_class->setID($class_id);
+    $accession = $details['term'];
+    if ($accession != $class_id) {
+      $supported_class->addContextItem($accession, $class_id);
+    }
+    $supported_class->setID($accession);
 
     // Set the class Type.
     if (array_key_exists('type', $details)) {

tripal_ws/includes/TripalWebService/TripalContentService_v0_1.inc

@@ -53,8 +53,32 @@ class TripalContentService_v0_1 extends TripalWebService {
 
     // is this a valid content type?
     if ($ctype) {
-      $bundle = tripal_load_bundle_entity(array('label' => $ctype));
-      if (!$bundle) {
+      // Get the list of published terms (these are the bundle IDs)
+      $bquery = db_select('tripal_bundle', 'tb');
+      $bquery->join('tripal_term', 'tt', 'tt.id = tb.term_id');
+      $bquery->join('tripal_vocab', 'tv', 'tv.id = tt.vocab_id');
+      $bquery->fields('tb', array('label'));
+      $bquery->fields('tt', array('accession'));
+      $bquery->fields('tv', array('vocabulary'));
+      $bquery->orderBy('tb.label', 'ASC');
+      $bundles = $bquery->execute();
+
+      // Iterate through the terms convert the santized name to the real label.
+      $i = 0;
+      $ctype_lookup = array();
+      $found = FALSE;
+      while ($bundle = $bundles->fetchObject()) {
+        if ($ctype == preg_replace('/[^\w]/', '_', $bundle->label)) {
+          $ctype = $bundle->label;
+          $found = TRUE;
+        }
+        if ($ctype == $bundle->vocabulary . ':' . $bundle->accession) {
+          $ctype = $bundle->label;
+          $found = TRUE;
+        }
+      }
+
+      if (!$found) {
         throw new Exception('Invalid content type: ' . $ctype);
       }
     }
@@ -62,7 +86,7 @@ class TripalContentService_v0_1 extends TripalWebService {
     // If we have a content type then list all of the entities that belong
     // to it.
     if ($ctype and !$entity_id and !$expfield) {
-      $this->doContentTypeList($ctype);
+      $this->doEntityList($ctype);
     }
     // If we have an entity ID then build the resource for a single entity.
     else if ($ctype and $entity_id and !$expfield) {
@@ -73,7 +97,7 @@ class TripalContentService_v0_1 extends TripalWebService {
     }
     // Otherwise just list all of the available content types.
     else {
-      $this->doAllTypesList();
+      $this->doContentTypesList();
     }
   }
 
@@ -118,7 +142,7 @@ class TripalContentService_v0_1 extends TripalWebService {
           FIELD_LOAD_CURRENT, array('field_id' => $field['id']));
     }
 
-    $this->addEntityField($term, $entity, $bundle, $field, $instance, $service_path, $expfield);
+    $this->addEntityField($this->resource, $term, $entity, $bundle, $field, $instance, $service_path, $expfield);
   }
 
   /**
@@ -139,7 +163,13 @@ class TripalContentService_v0_1 extends TripalWebService {
       $vocabulary = $instance['settings']['term_vocabulary'];
       $accession = $instance['settings']['term_accession'];
       $temp_term = tripal_get_term_details($vocabulary, $accession);
-      if ($temp_term['name'] == $expfield) {
+      // See if the name provided matches the field name after a bit of
+      // cleanup.
+      if (strtolower(preg_replace('/[^\w]/', '_', $temp_term['name'])) == strtolower($expfield)) {
+        return array($field, $instance, $temp_term);
+      }
+      // Alternatively if the CV term accession matches then we're good too.
+      if ($vocabulary . ':' . $accession == $expfield) {
         return array($field, $instance, $temp_term);
       }
     }
@@ -184,7 +214,7 @@ class TripalContentService_v0_1 extends TripalWebService {
     $this->addResourceProperty($this->resource, $itemPage, url('/bio_data/' . $entity->id, array('absolute' => TRUE)));
 
     // Add in the entitie's fields.
-    $this->addEntityFields($entity, $bundle, $term, $service_path);
+    $this->addEntityFields($this->resource, $entity, $bundle, $term, $service_path);
   }
 
   /**
@@ -213,7 +243,7 @@ class TripalContentService_v0_1 extends TripalWebService {
   /**
    * Adds the fields as properties of an entity resource.
    */
-  private function addEntityFields($entity, $bundle, $term, $service_path) {
+  private function addEntityFields($resource, $entity, $bundle, $term, $service_path) {
 
     // If the entity is set to hide fields that have no values then we
     // want to honor that in the web services too.
@@ -272,33 +302,33 @@ class TripalContentService_v0_1 extends TripalWebService {
         // that information.
         $items = field_get_items('TripalEntity', $entity, $field_name);
         $term_key = $this->getContextTerm($term, array('lowercase', 'spacing'));
-        $this->resource->addContextItem($term_key, array(
+        $resource->addContextItem($term_key, $vocabulary . ':' . $accession);
+        $resource->addContextItem($vocabulary . ':' . $accession, array(
           '@id' => $term['url'],
           '@type' => '@id'
         ));
         if ($items and count($items) > 0 and $items[0]['value']) {
-         $this->addResourceProperty($this->resource, $term, $service_path . '/' . $entity->id . '/' . urlencode($term['name']), array('lowercase', 'spacing'));
+          $this->addResourceProperty($resource, $term, $service_path . '/' . $entity->id . '/' . urlencode($term['name']), array('lowercase', 'spacing'));
         }
         else {
           if ($hide_fields == 'show') {
-            $this->addResourceProperty($this->resource, $term, NULL, array('lowercase', 'spacing'));
+            $this->addResourceProperty($resource, $term, NULL, array('lowercase', 'spacing'));
           }
         }
         continue;
       }
 
       // Get the details for this field for the JSON-LD response.
-      $this->addEntityField($term, $entity, $bundle, $field, $instance, $service_path);
+      $this->addEntityField($resource, $term, $entity, $bundle, $field, $instance, $service_path);
     }
   }
 
   /**
    * Adds the field as a property of the entity resource.
    */
-  private function addEntityField($term, $entity, $bundle, $field, $instance,
+  private function addEntityField($resource, $term, $entity, $bundle, $field, $instance,
       $service_path, $expfield = NULL) {
 
-
     // If the entity is set to hide fields that have no values then we
     // want to honor that in the web services too.
     $hide_fields = tripal_get_bundle_variable('hide_empty_field', $bundle->id, 'hide');
@@ -321,11 +351,11 @@ class TripalContentService_v0_1 extends TripalWebService {
     $values = array();
     for ($i = 0; $i < count($items); $i++) {
       if (array_key_exists('value', $items[$i])) {
-        $values[$i] = $this->sanitizeFieldKeys($this->resource, $items[$i]['value'], $bundle, $service_path);
+        $values[$i] = $this->sanitizeFieldKeys($resource, $items[$i]['value'], $bundle, $service_path);
       }
       elseif ($field['type'] == 'image') {
         $url = file_create_url($items[$i]['uri']);
-        $values[$i] = $this->sanitizeFieldKeys($this->resource, $url, $bundle, $service_path);
+        $values[$i] = $this->sanitizeFieldKeys($resource, $url, $bundle, $service_path);
       }
       else {
         // TODO: handle this case.
@@ -344,17 +374,17 @@ class TripalContentService_v0_1 extends TripalWebService {
       if (is_array($values[0])) {
         if ($expfield) {
           foreach ($values[0] as $k => $v) {
-            $this->resource->addProperty($k, $v);
+            $resource->addProperty($k, $v);
           }
         }
         else {
-          $this->addResourceProperty($this->resource, $term, $values[0], array('lowercase', 'spacing'));
+          $this->addResourceProperty($resource, $term, $values[0], array('lowercase', 'spacing'));
         }
       }
       // If the value is not an array it's a scalar so add it as is to the
       // response.
       else {
-        $this->addResourceProperty($this->resource, $term, $values[0], array('lowercase', 'spacing'));
+        $this->addResourceProperty($resource, $term, $values[0], array('lowercase', 'spacing'));
       }
     }
 
@@ -372,7 +402,7 @@ class TripalContentService_v0_1 extends TripalWebService {
         $member->setID($i);
         // Add the context of the parent resource because all of the keys
         // were santizied and set to match the proper context.
-        $member->setContext($this->resource);
+        $member->setContext($resource);
         $this->setResourceType($member, $term);
         foreach ($element as $key => $value) {
           $member->addProperty($key, $value);
@@ -381,11 +411,11 @@ class TripalContentService_v0_1 extends TripalWebService {
         $i++;
       }
       if ($expfield) {
-        $this->resource = $response;
+        $resource = $response;
       }
       else {
         //$this->resource->addProperty($key, $response);
-        $this->addResourceProperty($this->resource, $term, $response, array('lowercase', 'spacing'));
+        $this->addResourceProperty($resource, $term, $response, array('lowercase', 'spacing'));
       }
     }
   }
@@ -659,20 +689,15 @@ class TripalContentService_v0_1 extends TripalWebService {
    *
    * @throws Exception
    */
-  private function getFilters($field_mapping, $bundle) {
+  private function getFieldFilters($field_mapping, $bundle) {
     $filters = array();
 
     // Iterate through the paramter list provided by user.
     foreach ($this->params as $param => $value) {
 
       // Ignore non filter parameters.
-      if ($param == 'page' or $param == 'limit') {
-        continue;
-      }
-
-      // Ignore the order parameter as that is handled by the getOrderBy()
-      // function
-      if ($param == 'order') {
+      if ($param == 'page' or $param == 'limit' or $param == 'order' or
+          $param == 'ids' or $param == 'fields') {
         continue;
       }
 
@@ -779,8 +804,8 @@ class TripalContentService_v0_1 extends TripalWebService {
   /**
    * Creates a collection of resources for a given type.
    */
-  private function doContentTypeList($ctype) {
-    $service_path = $this->getServicePath() . '/' . urlencode($ctype);
+  private function doEntityList($ctype) {
+    $service_path = $this->getServicePath() . '/' . preg_replace('/[^\w]/', '_', $ctype);
     $this->resource = new TripalWebServiceCollection($service_path, $this->params);
 
     // Get the TripalBundle, TripalTerm and TripalVocab type for this type.
@@ -789,10 +814,11 @@ class TripalContentService_v0_1 extends TripalWebService {
     $term = reset($term);
 
     // The type of collection is provided by our API vocabulary service.
-    $vocab_service = new TripalVocabService_v0_1($this->base_path);
+    $vocab_service = new TripalDocService_v0_1($this->base_path);
     $this->resource->addContextItem('vocab', $vocab_service->getServicePath() . '#');
-    $this->resource->addContextItem(urlencode($bundle->label) . 'Collection', 'vocab:' . urlencode($bundle->label) . 'Collection');
-    $this->resource->setType(urlencode($bundle->label) . 'Collection');
+    $accession = preg_replace('/[^\w]/', '_', $bundle->label . ' Collection');
+    $this->resource->addContextItem($accession, 'vocab:' . $accession);
+    $this->resource->setType($accession);
 
     // Convert term to a simple array
     $term = tripal_get_term_details($term->vocab->vocabulary, $term->accession);
@@ -805,23 +831,34 @@ class TripalContentService_v0_1 extends TripalWebService {
     $field_mapping = $this->getFieldMapping($bundle);
 
     // Get arrays for filters and order by statements.
-    $filters = $this->getFilters($field_mapping, $bundle);
+    $filters = $this->getFieldFilters($field_mapping, $bundle);
     $order_by = $this->getOrderBy($field_mapping, $bundle);
 
-    // Initialize the query to search for records for out bundle type
+    // Initialize the query to search for records for our bundle types
     // that are published.
     $query = new TripalFieldQuery();
     $query->entityCondition('entity_type', 'TripalEntity');
     $query->entityCondition('bundle', $bundle->name);
     $query->propertyCondition('status', 1);
 
+    if (array_key_exists('ids', $this->params)) {
+      $eids = explode(',', $this->params['ids']);
+      if (count($eids) > 1000) {
+        throw new Exception('Please provide no more than 1000 ids.');
+      }
+      if (!is_numeric(implode('', $eids))) {
+        throw new Exception('All supplied ids must be numeric.');
+      }
+      $query->entityCondition('entity_id', $eids, 'IN');
+    }
+
     // Now iterate through the filters and add those.
     foreach ($filters as $key_field_name => $key_filters) {
       foreach ($key_filters as $i => $filter) {
          $column_name = $filter['column'];
          $value = $filter['value'];
          $op = $filter['op'];
-        $query->fieldCondition($key_field_name, $column_name, $value, $op);
+         $query->fieldCondition($key_field_name, $column_name, $value, $op);
       }
     }
 
@@ -865,6 +902,26 @@ class TripalContentService_v0_1 extends TripalWebService {
       $entity_ids = $results['TripalEntity'];
     }
 
+    // If the user wants to include any fields in the list then those provided
+    // names need to be converted to fields.
+    $add_fields = array();
+    $add_field_ids = array();
+    if (array_key_exists('fields', $this->params)) {
+      $fields = explode(',', $this->params['fields']);
+      foreach ($fields as $expfield) {
+        list($field, $instance, $temp_term) = $this->findField($bundle, $expfield);
+        if ($field) {
+          $add_fields[$expfield]['field'] = $field;
+          $add_fields[$expfield]['instance'] = $instance;
+          $add_fields[$expfield]['term'] = $temp_term;
+          $add_field_ids[] = $field['id'];
+        }
+        else {
+          throw new Exception(t('The field named, "!field", does not exist.', array('!field' => $expfield)));
+        }
+      }
+    }
+
     // Iterate through the entities and add them to the output list.
     foreach ($entity_ids as $entity_id => $stub) {
       // We don't need all of the attached fields for an entity so, we'll
@@ -884,6 +941,18 @@ class TripalContentService_v0_1 extends TripalWebService {
       $this->setResourceType($member, $term);
       $this->addResourceProperty($member, $label, $entity->title);
       $this->addResourceProperty($member, $itemPage, url('/bio_data/' . $entity->id, array('absolute' => TRUE)));
+
+      $entity = tripal_load_entity('TripalEntity', array($entity_id), FALSE, $add_field_ids);
+      $entity = $entity[$entity_id];
+
+      // Add in any requested fields
+      foreach ($fields as $expfield) {
+        if (array_key_exists($expfield, $add_fields)) {
+          $this->addEntityField($member, $add_fields[$expfield]['term'], $entity,
+              $bundle, $add_fields[$expfield]['field'], $add_fields[$expfield]['instance'],
+              $service_path);
+        }
+      }
       $this->resource->addMember($member);
     }
   }
@@ -891,13 +960,13 @@ class TripalContentService_v0_1 extends TripalWebService {
   /**
    * Creates a resources that contains the list of content types.
    */
-  private function doAllTypesList() {
+  private function doContentTypesList() {
     $service_path = $this->getServicePath();
-    $service_vocab = new TripalVocabService_v0_1($this->base_path);
+    $service_vocab = new TripalDocService_v0_1($this->base_path);
     $this->resource = new TripalWebServiceCollection($service_path, $this->params);
     $this->resource->addContextItem('vocab', $service_vocab->getServicePath());
-    $this->resource->addContextItem('ContentCollection', $service_vocab->getServicePath() . '#ContentCollection');
-    $this->resource->setType('ContentCollection');
+    $this->resource->addContextItem('Content_Collection', $service_vocab->getServicePath() . '#Content_Collection');
+    $this->resource->setType('Content_Collection');
 
     $label = tripal_get_term_details('rdfs', 'label');
     $this->addResourceProperty($this->resource, $label, 'Content Types');
@@ -919,12 +988,13 @@ class TripalContentService_v0_1 extends TripalWebService {
       $term = tripal_get_term_details($term->vocab->vocabulary, $term->accession);
 
       $member = new TripalWebServiceResource($service_path);
-      $member->setID(urlencode($bundle->label));
+      $member->setID(preg_replace('/[^\w]/', '_', $bundle->label));
 
-      $vocab_service = new TripalVocabService_v0_1($this->base_path);
+      $vocab_service = new TripalDocService_v0_1($this->base_path);
       $member->addContextItem('vocab', $vocab_service->getServicePath() . '#');
-      $member->addContextItem(urlencode($bundle->label) . 'Collection', 'vocab:' . urlencode($bundle->label) . 'Collection');
-      $member->setType(urlencode($bundle->label) . 'Collection');
+      $accession = preg_replace('/[^\w]/', '_', $bundle->label . ' Collection');
+      $member->addContextItem($accession, 'vocab:' . $accession);
+      $member->setType($accession);
 
       // Make sure the term has a URL.
       $url = $term['url'];
@@ -954,7 +1024,8 @@ class TripalContentService_v0_1 extends TripalWebService {
    */
   private function addDocContentCollectionClass() {
     $details = array(
-      'id' => 'vocab:ContentCollection',
+      'id' => 'vocab:Content_Collection',
+      'term' => 'vocab:Content_Collection',
       'title' => 'Content Collection',
     );
     $vocab = tripal_get_vocabulary_details('hydra');
@@ -1017,14 +1088,15 @@ class TripalContentService_v0_1 extends TripalWebService {
       // use the term definition
       $description = tripal_get_bundle_variable('description', $bundle->id);
       if (!$description) {
-        $description = $term->definition;
+        $description = $term->getDefinition();
       }
 
       // Create the details array for the class.
       $class_id = $this->getServicePath() . '/' . urlencode($bundle->label);
       $details = array(
-        'id' => $term->url,
-        'title' => $bundle->label,
+        'id' => $term->getURL(),
+        'term' => $term->getAccession(),
+        'title' => preg_replace('/[^\w]/', '_', $bundle->label),
         'description' => $description,
       );
 
@@ -1168,15 +1240,16 @@ class TripalContentService_v0_1 extends TripalWebService {
         $proptype = $link;
       }
 
-      $formatters = tripal_get_field_field_formatters($field);
+      $formatters = tripal_get_field_field_formatters($field, $instance);
+
       $property = array(
         'type' => $proptype,
         'title' => $instance['label'],
         'description' => $instance['description'],
-        'required' => $instance['required'] ? TRUE : FALSE,
-        'readonly' => FALSE,
-        'writeonly' => TRUE,
-        'tripal_formatters' => $formatters,
+        "required" => $instance['required'] ? TRUE : FALSE,
+        "readonly" => FALSE,
+        "writeonly" => TRUE,
+        "tripal_formatters" => $formatters,
       );
       $properties[] = $property;
     }
@@ -1186,8 +1259,12 @@ class TripalContentService_v0_1 extends TripalWebService {
    * Every content type (bundle) needs a collection class in the documentation.
    */
   private function addDocBundleCollectionClass($bundle, $term) {
+
+    $accession = preg_replace('/[^\w]/', '_', $bundle->label . ' Collection');
+
     $details = array(
-      'id' => 'vocab:' . urlencode($bundle->label) . 'Collection',
+      'id' => 'vocab:' . $accession,
+      'term' => 'vocab:' . $accession,
       'title' => $bundle->label . ' Collection',
       'subClassOf' => 'hydra:Collection',
       'description' => 'A collection (or list) of ' . $bundle->label . ' resources.',

tripal_ws/includes/TripalWebService/TripalVocabService_v0_1.inc → tripal_ws/includes/TripalWebService/TripalDocService_v0_1.inc

@@ -1,21 +1,21 @@
 <?php
 
-class TripalVocabService_v0_1 extends TripalWebService {
+class TripalDocService_v0_1 extends TripalWebService {
 
   /**
    * The human-readable label for this web service.
    */
-  public static $label = 'Vocabulary';
+  public static $label = 'API Documentation';
   /**
    * A bit of text to describe what this service provides.
    */
-  public static $description = 'Provides access to vocabulary terms that are in use by this site.';
+  public static $description = 'Provides Hydra style documenation to make this RESTful webservice discoverable.';
   /**
    * A machine-readable type for this service. This name must be unique
    * among all Tripal web services and is used to form the URL to access
    * this service.
    */
-  public static $type = 'vocab';
+  public static $type = 'doc';
 
   /**
    * The list of web services.
@@ -24,7 +24,7 @@ class TripalVocabService_v0_1 extends TripalWebService {
 
 
   /**
-   * Constructor for the TripalVocabService_v0_1 class.
+   * Constructor for the TripalDocService_v0_1 class.
    */
   public function __construct($base_path) {
     parent::__construct($base_path);
@@ -52,7 +52,7 @@ class TripalVocabService_v0_1 extends TripalWebService {
     $this->resource->addContextItem('apiDocumentation', 'hydra:apiDocumentation');
     $this->resource->addContextItem('supportedClass', 'hydra:supportedClass');
     $this->resource->setType('apiDocumentation');
-    $this->resource->setID('vocab/' . $this->getVersion());
+    $this->resource->setID('doc/' . $this->getVersion());
 
     // Add the EntryPoint class.
     $this->addEntryPointClass();
@@ -81,6 +81,7 @@ class TripalVocabService_v0_1 extends TripalWebService {
     $service_path = $this->getServicePath();
     $details = array(
       'id' => $service_path . '#EntryPoint',
+      'term' => 'vocab:EntryPoint',
       'title' => 'EntryPoint',
       'description' => 'The main entry point or homepage of the API',
       'subClassOf' => NULL,
@@ -134,7 +135,7 @@ class TripalVocabService_v0_1 extends TripalWebService {
       $op->addProperty('label', 'Retrieves the ' . $service_class::$label . ' resource.');
       $op->addProperty('description', NULL);
       $op->addProperty('expects', NULL);
-      $op->addProperty('returns', 'vocab:EntryPoint/' . $service::$type);
+      $op->addProperty('returns', 'local:EntryPoint/' . $service::$type);
       $op->addProperty('statusCodes', array());
       $ops[] = $op;
       $link->addContextItem('supportedOperation', 'hydra:supportedOperation');

tripal_ws/tripal_ws.module

@@ -39,8 +39,7 @@ require_once "includes/TripalFields/WebServicesFieldFormatter.inc";
 function tripal_ws_init() {
   global $base_url;
 
-  $version = 'v0.1';
-  $api_url = $base_url . '/ws/' . $version;
+  $api_url = $base_url . '/web-sevices/';
 
   $vocab = tripal_get_vocabulary_details('hydra');
 
@@ -49,7 +48,7 @@ function tripal_ws_init() {
   // This allows a hydra-enabled client to discover the API and use it.
   $attributes = array(
     'rel' => $vocab['sw_url'] . 'apiDocumentation',
-    'href' => $api_url . '/ws-doc/',
+    'href' => $api_url . '/doc/v0.1',
   );
   drupal_add_html_head_link($attributes, $header = FALSE);
 }
@@ -138,8 +137,8 @@ function tripal_ws_get_services() {
 
   // Add a link header for the vocabulary service so that clients
   // know where to find the docs.
-  tripal_load_include_web_service_class('TripalVocabService_v0_1');
-  $service = new TripalVocabService_v0_1($service_path);
+  tripal_load_include_web_service_class('TripalDocService_v0_1');
+  $service = new TripalDocService_v0_1($service_path);
   $vocab = tripal_get_vocabulary_details('hydra');
   drupal_add_http_header('Link', '<' . $service->getServicePath() . '>; rel="' . $vocab['sw_url'] . 'apiDocumentation"');
   drupal_add_http_header('Cache-Control', "no-cache");
@@ -176,7 +175,7 @@ function tripal_ws_get_services() {
     }
     // If the URL doesn't match then return not found.
     else {
-      throw new Exception("Unsupported service URL.  Web service URLs must be of the following format:  ");
+      throw new Exception("Unsupported service URL: '" . $ws_path[1] . "'");
     }
 
     // Get the service that matches the service_name
@@ -235,8 +234,8 @@ function tripal_ws_list_services() {
   $resource = new TripalWebServiceResource($base_path);
 
   // Add the vocabulary to the context.
-  tripal_load_include_web_service_class('TripalVocabService_v0_1');
-  $service = new TripalVocabService_v0_1($base_path);
+  tripal_load_include_web_service_class('TripalDocService_v0_1');
+  $service = new TripalDocService_v0_1($base_path);
   $resource->addContextItem('vocab', $service->getServicePath() . '#');
   $resource->addContextItem('EntryPoint', 'vocab:EntryPoint');
   $resource->setType('EntryPoint');
@@ -244,7 +243,7 @@ function tripal_ws_list_services() {
   // Now add the services as properties.
   foreach ($services as $service_class) {
     tripal_load_include_web_service_class($service_class);
-    if ($service_class == 'TripalVocabService_v0_1') {
+    if ($service_class == 'TripalDocService_v0_1') {
       continue;
     }
     $service = new $service_class($base_path);