Moved WS related functions from collections over to WS API. Working on performance improvements

tripal/includes/TripalEntityCollection.inc

@@ -36,7 +36,7 @@ class TripalEntityCollection {
   /**
    * The list of downloaders available for this bundle.
    */
-  protected $downloaders = array();
+  protected $formatters = array();
 
   /**
    * The description for this collection.
@@ -71,7 +71,7 @@ class TripalEntityCollection {
         ->execute();
 
       // Remove any files that may have been created
-      foreach ($this->downloaders as $class_name => $label) {
+      foreach ($this->formatters as $class_name => $label) {
         tripal_load_include_downloader_class($class_name);
         $outfile = $this->getOutfile($class_name);
         $downloader = new $class_name($this->collection_id, $outfile);
@@ -122,43 +122,30 @@ class TripalEntityCollection {
     $this->description = $collection->description;
     $this->collection_id = $collection->collection_id;
 
-    /* Add the IDs, Fields, Bundles for this collection from the
-     * collection_bundle table.
-     */
-    $this->bundles = $this->getBundles();
+    // Now get the bundles in this collection.
+    $bundles = db_select('tripal_collection_bundle', 'tcb')
+      ->fields('tcb')
+      ->condition('collection_id', $collection->collection_id)
+      ->execute();
+
     // If more than one bundle plop into associative array.
-    $bundle_name = "";
-    if (count($this->bundles) > 1) {
-      foreach ($this->bundles as $bundle) {
-        // If bundle name is not bio_data_# then it's an accession name from
-        // a remote site, so we need to handle it differently.
-        $bundle_name = $bundle->bundle_name;
-        if (strpos($bundle->bundle_name, 'bio_data_') !== 0) {
-          $ids[$bundle_name] = $this->getEntityIDs($bundle_name);
-          $fields[$bundle_name] = $this->getFieldIDs($bundle_name);
-        }
-        $bundle_name = $bundle->bundle_name;
-        $ids[$bundle_name] = $this->getEntityIDs($bundle_name);
-        $fields[$bundle_name] = $this->getFieldIDs($bundle_name);
-      }
-      $this->ids = $ids;
-      $this->fields = $fields;
-    }
-    else {
-      if (!empty($this->bundles)) {
-        $bundle_name = $this->bundles[0]->bundle_name;
-        $this->ids = $this->getEntityIDs($bundle_name);
-        $this->fields = $this->getFieldIDs($bundle_name);
-      }
+    while ($bundle = $bundles->fetchObject()) {
+      $bundle_name = $bundle->bundle_name;
+      $this->bundles[$bundle_name] = $bundle;
+      $this->ids[$bundle_name] = unserialize($bundle->ids);
+      $this->fields[$bundle_name] = unserialize($bundle->fields);
     }
+
     // Iterate through the fields and find out what download formats are
     // supported for this basket.
-    $this->downloaders = $this->getDownloadFormattersList($this->fields);
+    $this->formatters = $this->setFormatters();
   }
 
   /**
-   * Creates a new unique collection ID used as a look up against the
-   * tripal_collection_bundle to find fields, ids, and bundles.
+   * Creates a new data collection.
+   *
+   * To add bundles with entities and fields to a collection, use the
+   * addBundle() function after the collection is created.
    *
    * @param  $details
    *   An association array containing the details for a collection. The
@@ -199,8 +186,7 @@ class TripalEntityCollection {
           'description' => array_key_exists('description', $details) ? $details['description'] : '',
         ))
         ->execute();
-      // Now add the second table with bundle info.
-      $this->addFields($details, $collection_id);
+
       // Now load the job into this object.
       $this->load($collection_id);
     }
@@ -221,7 +207,7 @@ class TripalEntityCollection {
    *
    * @throws Exception
    */
-  public function addFields($details, $collection_id) {
+  public function addBundle($details, $collection_id) {
     if (!$details['bundle_name']) {
       throw new Exception("Must provide a 'bundle_name' to TripalEntityCollection::addFields().");
     }
@@ -241,8 +227,9 @@ class TripalEntityCollection {
           'collection_id' => $collection_id,
         ))
         ->execute();
+
       // Now load the job into this object.
-      //$this->load($collection_bundle_id);
+      $this->load($collection_id);
     }
     catch (Exception $e) {
       throw new Exception('Cannot create collection: ' . $e->getMessage());
@@ -256,100 +243,41 @@ class TripalEntityCollection {
    *   An array of bundles.
    */
   public function getBundles() {
-    $collection_id = $this->collection_id;
-    // Return the bundles from the collection_bundle table.
-    $result = db_select('tripal_collection_bundle')
-      ->fields('tripal_collection_bundle', array('bundle_name'))
-      ->condition('collection_id', $collection_id, '=')
-      ->execute()
-      ->fetchAll();
-
-    return $result;
+    return $this->bundles;
   }
 
   /**
    * Retrieves the site id for this specific bundle fo the collection.
    *
    * @return
-   *   A single site id.
-   */
-  public function getSiteId($bundle_name) {
-    $collection_id = $this->collection_id;
-    // Return the bundles from the collection_bundle table.
-    $result = db_select('tripal_collection_bundle')
-      ->fields('tripal_collection_bundle', array('site_id'))
-      ->condition('collection_id', $collection_id, '=')
-      ->condition('bundle_name', $bundle_name, '=')
-      ->execute()
-      ->fetchAssoc();
-
-    return $result;
-  }
-
-  /**
-   * Retrieves the vocabulary for a remote Tripal web service.
-   *
-   * @return
-   *   The vocabulary of a remote Tripal web service.
+   *   A remote site ID, or an empty string if the bundle is local.
    */
-  protected function retrieveRemoteAPIDoc($site_id) {
-    $cache_name = 'tripal_web_services_doc_' . $site_id;
-    if ($cache = cache_get($cache_name)) {
-      $site_doc = $cache->data;
-    }
-    else {
-      $site_doc = tripal_get_remote_site_doc($site_id);
-      if (!$site_doc) {
-        cache_set('tripal_web_services_doc_'  . $site_id, $site_doc);
-      }
-    }
-    return $site_doc;
+  public function getBundleSiteId($bundle_name) {
+    return $this->bundles[$bundle_name]->site_id;
   }
 
   /**
    * Retrieves the list of appropriate download formatters for the basket.
    *
-   * @param $fields
-   *   An array of field_ids, organized by bundles. They key is a bundle
-   *   name and the value is the list of fields for that bundle.
-   *
    * @return
    *   An associative array where the key is the TripalFieldDownloader class
    *   name and the value is the human-readable lable for the formatter.
    */
-  public function getDownloadFormattersList($fields) {
+  private function setFormatters() {
 
     $downloaders = array();
     // Iterate through the fields and find out what download formats are
     // supported for this basket.
-    foreach ($fields as $bundle_name => $field_group) {
-      foreach ($field_group as $field_id) {
-        // Check is $field_groups is an array because if it is that means we
-        // nested arrays we need to deal with.
-        if (is_array($field_id)) {
-          foreach ($field_id as $fid) {
-            // If the $field is numeric it's a field id from the local site but
-            // if it is not then it's a remote field that needs to be handled differently.
-            if (!is_numeric($fid)) {
-              // Need the site ID from the tripal_collection_bundle table.
-              $site_id = $this->getSiteId($bundle_name);
-              $site_doc = $this->retrieveRemoteAPIDoc($site_id);
-              // Now find the tripal formatters in the json data returned.
-              $this->getRemoteFieldDownloadFormats($site_doc, $bundle_name, $fid, $site_id['site_id']);
-            }
-            else {
-              $field_info = field_info_field_by_id($fid);
-              if (!$field_info) {
-                continue;
-              }
-              $field_name = $field_info['field_name'];
-              $instance = field_info_instance('TripalEntity', $field_name, $bundle_name);
-              // API function
-              // All fields should support the Tab and CSV downloaders.
-              $downloaders = array();
-              $this->downloaders += tripal_get_field_field_formatters($field_info, $instance);
-            }
-          }
+    foreach ($this->fields as $bundle_name => $field_ids) {
+
+      // Need the site ID from the tripal_collection_bundle table.
+      $site_id = $this->getBundleSiteId($bundle_name);
+
+      foreach ($field_ids as $field_id) {
+        // If this is a field from a remote site then get it's formatters
+        if ($site_id and module_exists('tripal_ws')) {
+          $formatters = tripal_get_remote_field_formatters($site_id, $bundle_name, $field_id);
+          $this->formatters += $formatters;
         }
         else {
           $field_info = field_info_field_by_id($field_id);
@@ -358,17 +286,15 @@ class TripalEntityCollection {
           }
           $field_name = $field_info['field_name'];
           $instance = field_info_instance('TripalEntity', $field_name, $bundle_name);
-          // API function
-          $downloaders = array();
-          $this->downloaders += tripal_get_field_field_formatters($field_info, $instance);
+          $formatters = tripal_get_field_field_formatters($field_info, $instance);
+          $this->formatters += $formatters;
         }
       }
     }
-    $this->downloaders = array_unique($this->downloaders);
-    return $this->downloaders;
+    $this->formatters = array_unique($this->formatters);
+    return $this->formatters;
   }
 
-
   /**
    * Retrieves the list of appropriate download formatters for the basket.
    *
@@ -376,80 +302,8 @@ class TripalEntityCollection {
    *   An associative array where the key is the TripalFieldDownloader class
    *   name and the value is the human-readable lable for the formatter.
    */
-  public function getDownloadFormatters() {
-     return $this->downloaders;
-  }
-
-  /**
-   * Retrieves the list of remote download formatters for the basket and
-   * assigns them to $this->downloader.
-   *
-   */
-  public function getRemoteFieldDownloadFormats($site_doc, $bundle_name, $field, $site_id) {
-    // Now find the tripal formatters in the json data returned.
-    foreach ($site_doc as $item) {
-      if (is_array($item)) {
-        foreach ($item as $vocab_term) {
-          /* The returned $vocab item should look like this:
-          "supportedProperty": [
-          {
-          "property": "OBI:0100026",
-          "hydra:title": "Organism",
-          "hydra:description
-          http://www.w3.org/ns/hydra/core#description
-          ": "The full scientific name for a species..",
-          "required": false,
-          "readable": false,
-          "writeable": true.
-          "tripal_formatter": "
-          (
-          [0] => TripalTabDownloader
-          [1] => TripalCSVDownloader
-          )"
-          }, */
-          if (!empty($vocab_term['supportedProperty'])) {
-            $vocab_supported_properties = $vocab_term['supportedProperty'];
-            if (is_array($vocab_supported_properties)) {
-              foreach ($vocab_supported_properties as $property) {
-                if ($property['property'] === $field) {
-                  if (array_key_exists('tripal_formatters', $property)) {
-                    $download_types = $property['tripal_formatters'];
-                    if (is_array($download_types)) {
-                      foreach ($download_types as $download_type) {
-                        $this->downloaders[$download_type] = $download_type;
-                        return $this->downloaders;
-                      }
-                    }
-                    else {
-                      $this->downloaders[$download_types] = $download_types;
-                      return $this->downloaders;
-                    }
-                  }
-                  else {
-                    $result = db_select('tripal_sites')
-                      ->fields('tripal_sites', array('name'))
-                      ->condition('id', $site_id, '=')
-                      ->execute()
-                      ->fetchAssoc();
-                    $site_name = $result['name'];
-                    drupal_set_message(t("Cannot find tripal_formatters array elements in the remote site: $site_name"), 'warning');
-                  }
-                }
-              }
-            }
-            else {
-              if (in_array('tripal_formatters', $vocab_supported_properties)) {
-                $download_types = $vocab_supported_properties['tripal_formatters'];
-                foreach ($download_types as $download_type) {
-                  $this->downloaders[$download_type] = $download_type;
-                  return $this->downloaders;
-                }
-              }
-            }
-          }
-        }
-      }
-    }
+  public function getFormatters() {
+    return $this->formatters;
   }
 
   /**
@@ -459,22 +313,7 @@ class TripalEntityCollection {
    *   An array of numeric entity IDs.
    */
   public function getEntityIDs($bundle_name) {
-    $entity_ids = array();
-
-    if (!$bundle_name) {
-      throw new Exception('Please provide the $bundle_name argument for the TripalEntityCollection::getEntityIDs() function.');
-    }
-
-    // Get the IDs for the entities for tihs bundle collection.
-    $collection_id = $this->collection_id;
-    $ids = db_select('tripal_collection_bundle')
-      ->fields('tripal_collection_bundle', array('ids'))
-      ->condition('collection_id', $collection_id, '=')
-      ->condition('bundle_name', $bundle_name, '=')
-      ->execute()
-      ->fetchField();
-
-    return unserialize($ids);
+    return $this->ids[$bundle_name];
   }
 
   /**
@@ -484,27 +323,7 @@ class TripalEntityCollection {
    *   An array of numeric field IDs.
    */
   public function getFieldIDs($bundle_name) {
-    $field_ids = array();
-
-    // Get the IDs for the fields for this bundle collection.
-    $collection_id = $this->collection_id;
-    $result = db_select('tripal_collection_bundle')
-      ->fields('tripal_collection_bundle', array('fields'))
-      ->condition('collection_id', $collection_id, '=')
-      ->condition('bundle_name', $bundle_name, '=')
-      ->execute()
-      ->fetchAll();
-
-    // Unserialize the array of standard class objects.
-    $unserialized_result = array();
-    foreach ($result as $field_list) {
-      $unserialized_field_list = unserialize($field_list->fields);
-      foreach ($field_list as $item) {
-        $unserialized_result[] = $unserialized_field_list;
-      }
-    }
-
-    return $unserialized_result;
+    return $this->fields[$bundle_name];
   }
 
   /**
@@ -608,8 +427,8 @@ class TripalEntityCollection {
    * @return boolean
    *   TRUE if the formatter is compatible, FALSE otherwise.
    */
-  public function isFormatterCompatible($formatter) {
-    foreach ($this->downloaders as $class_name => $label) {
+  protected function isFormatterCompatible($formatter) {
+    foreach ($this->formatters as $class_name => $label) {
       if ($class_name == $formatter) {
         return TRUE;
       }
@@ -636,11 +455,9 @@ class TripalEntityCollection {
    *   The name of the class
    */
   public function getOutfilePath($formatter) {
-    if(!$this->isFormatterCompatible($formatter)) {
+    if (!$this->isFormatterCompatible($formatter)) {
       throw new Exception(t('The formatter, "@formatter", is not compatible with this data collection.', array('@formatter' => $formatter)));
-
     }
-
     if (!tripal_load_include_downloader_class($formatter)) {
       throw new Exception(t('Cannot find the formatter named "@formatter".', array('@formatter', $formatter)));
     }
@@ -667,11 +484,9 @@ class TripalEntityCollection {
    */
   public function write($formatter = NULL, TripalJob $job = NULL) {
 
-    $downloaders = array();
-
     // Initialize the downloader classes and initialize the files for writing.
-    $formatters = $this->getDownloadFormatters();
-    foreach ($formatters as $class => $label) {
+    $formatters = array();
+    foreach ($this->formatters as $class => $label) {
       if (!$this->isFormatterCompatible($class)) {
         throw new Exception(t('The formatter, "@formatter", is not compatible with this data collection.', array('@formatter' => $formatter)));
       }
@@ -680,8 +495,8 @@ class TripalEntityCollection {
       }
       $outfile = $this->getOutfile($class);
       if (!$formatter or ($formatter == $class)) {
-        $downloaders[$class] = new $class($this->collection_id, $outfile);
-        $downloaders[$class]->writeInit($job);
+        $formatters[$class] = new $class($this->collection_id, $outfile);
+        $formatters[$class]->writeInit($job);
         if ($job) {
           $job->logMessage("Writing " . lcfirst($class::$full_label) . " file.");
         }
@@ -717,8 +532,8 @@ class TripalEntityCollection {
         // if we have a site_id then we need to get the entity from the
         // remote service. Otherwise create the entity from the local system.
         $entity = NULL;
-        if ($site_id) {
-          $entity = $this->loadRemoteEntity($entity_id, $site_id, $bundle_name);
+        if ($site_id and module_exists('tripal_ws')) {
+          $entity = tripal_load_remote_entity($entity_id, $site_id, $bundle_name, $field_ids);
           if (!$entity) {
             continue;
           }
@@ -733,115 +548,19 @@ class TripalEntityCollection {
         }
 
         // Write the same entity to all the formatters that are supported.
-        foreach ($downloaders as $my_formatter => $downloader) {
-          $downloader->writeEntity($entity, $job);
+        foreach ($formatters as $class => $formatter) {
+          if ($class == 'TripalTabDownloader') {
+            $formatter->writeEntity($entity, $job);
+          }
         }
       }
     }
 
     // Now close up all the files
-    foreach ($downloaders as $my_formatter => $downloader) {
-      $downloader->writeDone($job);
+    foreach ($formatters as $class => $formatter) {
+      $formatter->writeDone($job);
     }
   }
 
-  /**
-   * Build and return a fake entity from a remote site using
-   * tripal web services calls.
-   *
-   * @param $remote_ids
-   *   Array of the remote ids.
-   *
-   * @param $site_id
-   *   Unique site id assigned in the tripal_sites table when
-   *   a new site is created via the web services interface.
-   *
-   * @param $remote_fields
-   *   Array of the remote fields.
-   *
-   * @param $bundle_name
-   *   Bundle name of the remote field, in this instance it will be
-   *   the accession of the field.
-   *
-   * @return $fake_tripal_entity
-   *    This is a fake entity structured to allow the format
-   *    entity function to process and return the info.
-   */
-  protected function loadRemoteEntity($remote_id, $site_id, $bundle_name) {
-
-    // Get the site documentation
-    $site = empty($site_id) ? 'local' : $site_id;
-    $site_doc = $this->retrieveRemoteAPIDoc($site_id);
-
-    // Get the remote entity and create the fake entity.
-    $query = $bundle_name . '/' . $remote_id;
-    $remote_entity = tripal_query_remote_site($site_id, $query);
-    if (!$remote_entity) {
-      return FALSE;
-    }
-
-    // Start building the fake id.
-    $entity = new stdClass();
-    $entity->entityType = 'TripalEntity';
-    $entity->entityInfo = [];
-    $entity->id = $remote_id;
-    $entity->type = 'TripalEntity';
-    $entity->bundle = $bundle_name;
-    $entity->site_id = $site_id;
-
-    // Get the context JSON for this remote entity, we'll use it to map
-    // the properties to the correct fields.
-    $context = drupal_http_request($remote_entity['@context']);
-    $context = drupal_json_decode($context->data);
-    $context = $context['@context'];
-
-    // Iterate through the fields that are printable and get those values
-    // from the results.
-    foreach ($this->printable_fields as $accession => $label) {
-      $field_id = $this->fields2terms[$site][$bundle_name]['by_accession'][$accession];
-
-      // If the field isn't part of this bundle then skip it.
-      if (!$field_id) {
-        continue;
-      }
-
-      $field = $this->fields[$site][$bundle_name][$field_id]['field'];
-      $instance = $this->fields[$site][$bundle_name][$field_id]['instance'];
-      $field_name = $field['field_name'];
-      $accession = $instance['settings']['term_vocabulary'] . ':' . $instance['settings']['term_accession'];
 
-      // Get the key for this field from the context.
-      $field_key = $accession;
-      foreach ($context as $k => $v) {
-        if (!is_array($v)) {
-        }
-        if (!is_array($v) and $v == $accession) {
-          $field_key = $k;
-        }
-      }
-
-      // If the field is not in this remote bundle then add an empty value for
-      // it.
-      if (!$field_key) {
-        $entity->{$field_name}['und'][0]['value'] = '';
-        continue;
-      }
-
-      // If the key is for a field that is not "auto attached' then we need
-      // to get that field through a separate call.
-      $needs_query = FALSE;
-      if (array_key_exists($field_name, $context) and is_array($context[$field_name]) and
-          array_key_exists('@type', $context[$field_name]) and $context[$field_name]['@type'] == '@id'){
-            $needs_query = TRUE;
-      }
-
-      $value = '';
-      if (!$needs_query) {
-        $value = $remote_entity[$field_key];
-      }
-      $entity->{$field_name}['und'][0]['value'] = $value;
-    }
-
-    return $entity;
-  }
 }

tripal/includes/TripalFieldDownloaders/TripalFieldDownloader.inc

@@ -372,27 +372,6 @@ abstract class TripalFieldDownloader {
   }
 
 
-
-  /**
-   * Retrieves the vocabulary for a remote Tripal web service.
-   *
-   * @return
-   *   The vocabulary of a remote Tripal web service.
-   */
-  protected function retrieveRemoteAPIDoc($site_id) {
-    $cache_name = 'tripal_web_services_doc_' . $site_id;
-    if ($cache = cache_get($cache_name)) {
-      $site_doc = $cache->data;
-    }
-    else {
-      $site_doc = tripal_get_remote_site_doc($site_id);
-      if (!$site_doc) {
-        cache_set('tripal_web_services_doc_'  . $site_id, $site_doc);
-      }
-    }
-    return $site_doc;
-  }
-
   /**
    * A helper function for the setFields() function.
    *
@@ -401,16 +380,14 @@ abstract class TripalFieldDownloader {
   private function setLocalFields() {
     foreach ($this->collection_bundles as $collection_bundle) {
       $bundle_name = $collection_bundle->bundle_name;
-      $site = empty($collection_bundle->site_id) ? 'local' : $collection_bundle->site_id;
-      // Skip remote fields.
       if ($collection_bundle->site_id) {
         continue;
       }
       foreach ($collection_bundle->fields as $field_id) {
         $field = field_info_field_by_id($field_id);
         $instance = field_info_instance('TripalEntity', $field['field_name'], $bundle_name);
-        $this->fields[$site][$bundle_name][$field_id]['field'] = $field;
-        $this->fields[$site][$bundle_name][$field_id]['instance'] = $instance;
+        $this->fields['local'][$bundle_name][$field_id]['field'] = $field;
+        $this->fields['local'][$bundle_name][$field_id]['instance'] = $instance;
       }
     }
   }
@@ -421,72 +398,28 @@ abstract class TripalFieldDownloader {
    * Adds remote fields to the list of fields.
    */
   private function setRemoteFields() {
+    // We can't use the Tripal ws API extensions if the
+    // tripal_ws module is not enabled.
+    if (!module_exists('tripal_ws')) {
+      return;
+    }
+
     foreach ($this->collection_bundles as $collection_bundle) {
       $bundle_name = $collection_bundle->bundle_name;
+      $site_id = $collection_bundle->site_id;
       // Skip local fields.
-      if (!$collection_bundle->site_id) {
+      if (!$site_id) {
         continue;
       }
 
-      $site = empty($site_id) ? 'local' : $site_id;
-      $site_doc = $this->retrieveRemoteAPIDoc($collection_bundle->site_id);
-
-      // Get the class that matches this bundle.
-      $class = $this->getRemoteClass($bundle_name, $site_doc);
-
       // Iterate through the fields of this collection and get the
       // info for each one from the class.  We will create "fake" field and
       // instance info arrays.
       foreach ($collection_bundle->fields as $field_id) {
-
-        // Get the property from the document for this field.
-        $property = $this->getRemoteClassProperty($class, $field_id, $site_doc);
-
-        // Now create the fake field and instance.
-        list($vocab, $accession) = explode(':', $field_id);
-        $field_name = 'tripal_remote_site_' . $collection_bundle->site_id . '_' . $field_id;
-        $field = array(
-          'field_name' => $field_name,
-          'type' => $field_name,
-          'storage' => array(
-            'type' => 'tripal_remote_site'
-          ),
-        );
-        $instance = array(
-          'label' => $property['hydra:title'],
-          'description' => $property['hydra:description'],
-          'formatters' => $property['tripal_formatters'],
-          'settings' => array(
-            'term_vocabulary' => $vocab,
-            'term_accession' => $accession
-          ),
-          'field_name' => $field_name,
-          'entity_type' => 'TripalEntity',
-          'bundle_name' => $bundle_name,
-        );
-        $this->fields[$site][$bundle_name][$field_id]['field'] = $field;
-        $this->fields[$site][$bundle_name][$field_id]['instance'] = $instance;
-      }
-    }
-  }
-
-  private function getRemoteClass($class_id, $site_doc){
-    // Get the class that matches this bundle.
-    $classes = $site_doc['supportedClass'];
-    $class = NULL;
-    foreach ($classes as $item) {
-      if ($item['@id'] == $class_id) {
-        $class = $item;
-      }
-    }
-    return $class;
-  }
-
-  private function getRemoteClassProperty($class, $prop_id, $site_doc){
-    $properties = $class['supportedProperty'];
-    foreach ($properties as $item) {
-      if ($item['property'] == $prop_id) {
-         return $item;
+        $field = tripal_get_remote_field_info($site_id, $bundle_name, $field_id);
+        $instance = tripal_get_remote_field_instance_info($site_id, $bundle_name, $field_id);
+        $this->fields[$site_id][$bundle_name][$field_id]['field'] = $field;
+        $this->fields[$site_id][$bundle_name][$field_id]['instance'] = $instance;
       }
     }
   }

tripal/includes/TripalFieldDownloaders/TripalNucFASTADownloader.inc

@@ -84,7 +84,7 @@ class TripalNucFASTADownloader extends TripalFieldDownloader {
         }
         // Add in the organism.
         else if ($accession == 'OBI:0100026') {
-          $others[$instance['label']] = strip_tags($value['rdfs:label']);
+          //$others[$instance['label']] = strip_tags($value['rdfs:label']);
         }
         // All other fields add them to the others list.
         else {

tripal/includes/TripalFieldDownloaders/TripalTabDownloader.inc

@@ -62,6 +62,7 @@ class TripalTabDownloader extends TripalFieldDownloader {
            $row[] = $value;
          }
          else {
+           print_r($value);
            if (array_key_exists('rdfs:label', $entity->{$field_name}['und'][0]['value'])) {
              $label = $entity->{$field_name}['und'][0]['value']['rdfs:label'];
              $row[] = strip_tags($label);

tripal/includes/tripal.collections.inc

@@ -22,7 +22,7 @@ function tripal_user_collections_page() {
     $collection->load($collection_id);
 
     $downloads = array();
-    $formatters = $collection->getDownloadFormatters();
+    $formatters = $collection->getFormatters();
 
     foreach ($formatters as $class_name => $label) {
 

tripal_ws/api/tripal_ws.api.inc

@@ -186,14 +186,25 @@ function tripal_remove_site($record_id) {
 }
 
 /**
- * Retreives the full Hydra documentattion for the remote Tripal site.
+ * Makes a request to the "content"service of a remote Tripal web site.
  *
- * When passed a site_id from the tripal_sites table the full
- * site's document service is returned in json format
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ *
+ * @param $query
+ *   The query string. This string is added to the URL for the remote
+ *   website.
+ *
+ * @return
+ *   The JSON response formatted in a PHP array or FALSE if a problem occured.
  */
-function tripal_get_remote_site_doc($site_id) {
+function tripal_query_remote_site($site_id, $query) {
+  $ctype = $query;
+  $qdata = '';
+  if (preg_match('/\?/', $query)) {
+    list($ctype, $qdata) = explode('?', $query);
+  }
 
-  // Get the site url from the tripal_sites table.
   if ($site_id) {
     $remote_site = db_select('tripal_sites', 'ts')
       ->fields('ts')
@@ -206,10 +217,12 @@ function tripal_get_remote_site_doc($site_id) {
   $ws_version = $remote_site->version;
   $ws_url = $remote_site->url;
   $ws_url = trim($ws_url, '/');
-  $ws_url .= '/web-services/doc/' . $ws_version;
-
-  // Build and make the request.
-  $options = [];
+  $ws_url .= '/web-services/content/' . $ws_version . '/' . $ctype;
+  // Build the Query and make and substitions needed.
+  //dpm($ws_url . '?' . $query);
+  $options = array(
+    'data' => $qdata,
+  );
   $data = drupal_http_request($ws_url, $options);
 
   if (!$data) {
@@ -222,8 +235,8 @@ function tripal_get_remote_site_doc($site_id) {
   // connection error (not a Tripal web servcies error).
   if (property_exists($data, 'error')) {
     tripal_report_error('tripal_ws', TRIPAL_ERROR,
-        'Web Services error on remote site: %error.',
-        array('%error' => $data->error));
+        'Remote web Services reports the following error: !error. Using URL: !url',
+        array('!error' => $error, '!url' => $ws_url));
     return FALSE;
   }
 
@@ -234,8 +247,8 @@ function tripal_get_remote_site_doc($site_id) {
   if (array_key_exists('error', $data)) {
     $error = '</pre>' . print_r($data['error'], TRUE) . '</pre>';
     tripal_report_error('tripal_ws', TRIPAL_ERROR,
-        'Web Services error on remote site: %error.',
-        array('%error' => $error));
+        'Tripal remote web services reports the following error: !error. Using URL: !url',
+        array('!error' => $error, '!url' => $ws_url));
     return FALSE;
   }
 
@@ -243,40 +256,113 @@ function tripal_get_remote_site_doc($site_id) {
 }
 
 /**
- * Makes a request to a remote Tripal web services site.
+ * Retrieves the JSON-LD context for any remote Tripal web service.
  *
- * @param $query
- *   The query string. This string is added to the URL for the remote
- *   website.
+ * @param $context_url
+ *   The Full URL for the context file on the remote Tripal site. This URL
+ *   can be found in the '@context' key of any response from a remote Tripal
+ *   web services call.
+ * @param $cache_id
+ *   A unique ID for caching of this context result to speed furture
+ *   queries.
+ * @return
+ *   The JSON-LD context mapping array, or FALSE if the context could not
+ *   be retrieved.
+ */
+function tripal_get_remote_context($context_url, $cache_id) {
+
+  if (!$context_url) {
+    throw new Exception('PLease provide a context_url for the tripal_get_remote_context function.');
+  }
+  if (!$cache_id) {
+    throw new Exception('PLease provide unique $cache_id for the tripal_get_remote_context function.');
+  }
+
+  if ($cache = cache_get($cache_id)) {
+    return $cache->data;
+  }
+
+  $context = drupal_http_request($context_url);
+  if (!$context) {
+    tripal_report_error('tripal_ws', TRIPAL_ERROR,
+        'There was a poblem retrieving the context from the remote site: !context_url.',
+        array('!context_url' => $context_url));
+    return FALSE;
+  }
+  $context = drupal_json_decode($context->data);
+  $context = $context['@context'];
+  cache_set($cache_id, $context);
+  return $context;
+}
+
+/**
+ * Retrieves the JSON-LD context for a bundle or field on a remote Tripal site.
+ *
+ * The $site_id, $bundle_accession and $field_accession variables are not
+ * needed to retrieve the context, but are used for caching the context to
+ * make subsequent calls execute faster.  This function is meant to be used
+ * only for the 'content' service provided by Tripal.
  *
+ * @param $site_id
+ *    The numeric site ID for the remote Tripal site.
+ * @param $context_url
+ *   The Full URL for the context file on the remote Tripal site. This URL
+ *   can be found in the '@context' key of any response from a remote Tripal
+ *   web services call.
+ * @param $bundle_accession
+ *   The controlled vocabulary term accession for the content type
+ *   on the remote Tripal site.
+ * @param $field_accession
+ *   The controlled vocabulary term accession for the property (i.e. field) of
+ *   the Class (i.e. content type).
  * @return
- *   The JSON response formatted in a PHP array or FALSE if a problem occured.
+ *   The JSON-LD context mapping array.
  */
-function tripal_query_remote_site($site_id, $query) {
-  $ctype = $query;
-  $qdata = '';
-  if (preg_match('/\?/', $query)) {
-    list($ctype, $qdata) = explode('?', $query);
+function tripal_get_remote_content_context($site_id, $context_url, $bundle_accession, $field_accession = '') {
+  $cache_id = substr('trp_ws_context_' . $site_id . '-' . $bundle_accession . '-' . $field_accession, 0, 254);
+  $context = tripal_get_remote_context($context_url, $cache_id);
+  return $context;
+}
+/**
+ * Retrieves the API documentation for a remote Tripal web service.
+ *
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @return
+ *   The vocabulary of a remote Tripal web service, or FALSE if an error
+ *   occured.
+ */
+function tripal_get_remote_API_doc($site_id) {
+  $site_doc = '';
+
+  if (!$site_id) {
+    throw new Exception('PLease provide a numeric site ID for the tripal_get_remote_API_doc function.');
   }
 
-  if ($site_id) {
-    $remote_site = db_select('tripal_sites', 'ts')
-      ->fields('ts')
-      ->condition('ts.id', $site_id)
-      ->execute()
-      ->fetchObject();
+  $cache_name = 'trp_ws_doc_' . $site_id;
+  if ($cache = cache_get($cache_name)) {
+    return $cache->data;
+  }
+
+  // Get the site url from the tripal_sites table.
+  $remote_site = db_select('tripal_sites', 'ts')
+    ->fields('ts')
+    ->condition('ts.id', $site_id)
+    ->execute()
+    ->fetchObject();
+
+  if (!$remote_site) {
+    throw new Exception(t('Cannot find a remote site with id: "!id"', array('!id' => $site_id)));
   }
 
   // Build the URL to the remote web services.
   $ws_version = $remote_site->version;
   $ws_url = $remote_site->url;
   $ws_url = trim($ws_url, '/');
-  $ws_url .= '/web-services/content/' . $ws_version . '/' . $ctype;
-  // Build the Query and make and substitions needed.
-  //dpm($ws_url . '?' . $query);
-  $options = array(
-    'data' => $qdata,
-  );
+  $ws_url .= '/web-services/doc/' . $ws_version;
+
+  // Build and make the request.
+  $options = [];
   $data = drupal_http_request($ws_url, $options);
 
   if (!$data) {
@@ -289,22 +375,392 @@ function tripal_query_remote_site($site_id, $query) {
   // connection error (not a Tripal web servcies error).
   if (property_exists($data, 'error')) {
     tripal_report_error('tripal_ws', TRIPAL_ERROR,
-        'Web Services error on remote site: %error.',
-        array('%error' => $data->error));
+        'Remote web services document reports the following error: !error. Using URL: !url',
+      array('!error' => $error, '!url' => $ws_url));
     return FALSE;
   }
 
   // We got a response, so convert it to a PHP array.
-  $data = drupal_json_decode($data->data);
+  $site_doc = drupal_json_decode($data->data);
 
   // Check if there was a Tripal Web Services error.
   if (array_key_exists('error', $data)) {
     $error = '</pre>' . print_r($data['error'], TRUE) . '</pre>';
     tripal_report_error('tripal_ws', TRIPAL_ERROR,
-        'Web Services error on remote site: %error.',
-        array('%error' => $error));
+        'Tripal Remote web services document reports the following error: !error. Using URL: !url',
+      array('!error' => $error, '!url' => $ws_url));
     return FALSE;
   }
 
-  return $data;
+  cache_set($cache_name, $site_doc);
+
+  return $site_doc;
+}
+
+/**
+ * Build and return a "fake" entity for a record from a remote Tripal site.
+ *
+ * @param $remote_entity_id
+ *   Array of the remote ids.
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @param $bundle_accession
+ *   The controlled vocabulary term accession for the content type
+ *   on the remote Tripal site.
+ * @param $field_ids
+ *   The controlled vocabulary term accessions for the fields available
+ *   on the remote content type.  Any remote fields that matches these IDs will
+ *   be added to the entity returned.
+ * @return
+ *    A fake entity object.
+ */
+function tripal_load_remote_entity($remote_entity_id, $site_id, $bundle_accession, $field_ids) {
+
+  // Get the site documentation (loads from cache if already retrieved).
+  $site_doc = tripal_get_remote_API_doc($site_id);
+
+  // Get the remote entity and create the fake entity.
+  $query = $bundle_accession . '/' . $remote_entity_id;
+  $remote_entity = tripal_query_remote_site($site_id, $query);
+  if (!$remote_entity) {
+    return FALSE;
+  }
+
+  // Start building the fake id.
+  $entity = new stdClass();
+  $entity->entityType = 'TripalEntity';
+  $entity->entityInfo = [];
+  $entity->id = $remote_entity_id;
+  $entity->type = 'TripalEntity';
+  $entity->bundle = $bundle_accession;
+  $entity->site_id = $site_id;
+
+
+  // Get the context JSON for this remote entity, we'll use it to map
+  $context_url = $remote_entity['@context'];
+  $context = tripal_get_remote_content_context($site_id, $context_url, $bundle_accession);
+  if (!$context) {
+    return $entity;
+  }
+
+  // Iterate through the fields and the those values to the entity.
+  foreach ($field_ids as $field_id) {
+
+    $field = tripal_get_remote_field_info($site_id, $bundle_accession, $field_id);
+    $instance = tripal_get_remote_field_instance_info($site_id, $bundle_accession, $field_id);
+    $field_name = $field['field_name'];
+
+    $field_key = '';
+    foreach ($context as $k => $v) {
+      if (!is_array($v) and $v == $field_id) {
+        $field_key = $k;
+      }
+    }
+
+    // If the field is not in this remote bundle then add an empty value.
+    if (!$field_key) {
+      $entity->{$field_name}['und'][0]['value'] = '';
+      continue;
+    }
+    if (!array_key_exists($field_key, $remote_entity)) {
+      $entity->{$field_name}['und'][0]['value'] = '';
+      continue;
+    }
+
+    // If the key is for a field that is not "auto attached' then we need
+    // to get that field through a separate call.
+    $attached = TRUE;
+    if (array_key_exists($field_id, $context) and is_array($context[$field_id]) and
+        array_key_exists('@type', $context[$field_id]) and $context[$field_id]['@type'] == '@id'){
+          $attached = FALSE;
+    }
+
+    // Set the value for this field.
+    $value = '';
+    if (is_array($remote_entity[$field_key])) {
+      $value = _tripal_update_remote_entity_field($remote_entity[$field_key], $context, 1);
+    }
+    else {
+      $value = $remote_entity[$field_key];
+    }
+
+    // If the field is not attached then we have to query another level.
+    if (!$attached) {
+
+      $field_data = drupal_http_request($value);
+      if (!$field_data) {
+        tripal_report_error('tripal_ws', TRIPAL_ERROR,
+           'There was a poblem retrieving the unattached field, "!field:", for the remote entity: !entity_id.',
+            array('!field' => $field_id, '!entity_id' => $remote_entity_id));
+        $value = '';
+      }
+      $field_data = drupal_json_decode($field_data->data);
+
+      // Get the context for this field so we can map the keys to the
+      // controlled vocabulary accessions. If it fails then skip this field.
+      $field_context_url = $field_data['@context'];
+      $field_context = tripal_get_remote_content_context($site_id, $field_context_url, $bundle_accession, $field_id);
+      if (!$field_context) {
+        continue;
+      }
+      $value = _tripal_update_remote_entity_field($field_data, $field_context);
+    }
+    $entity->{$field_name}['und'][0]['value'] = $value;
+  }
+  return $entity;
+}
+
+/**
+ * A helper function for the tripal_get_remote_entity() function.
+ *
+ * This function converts the field's key elements to their
+ * vocabulary term accessions.
+ *
+ * @param $field_data
+ *   The field array as returned by web services.
+ * @param $context
+ *   The web service JSON-LD context for the bundle to which the field belongs.
+ */
+function _tripal_update_remote_entity_field($field_data, $context, $depth = 0) {
+
+  // If depth is 0 then we are tring to find out how to deal with this field.
+  if ($depth == 0) {
+    // Check if this is an array.
+    if ($field_data['@type'] == 'Collection') {
+      $members = array();
+      foreach ($field_data['member'] as $member) {
+        $next_depth = $depth + 1;
+        $members[] = _tripal_update_remote_entity_field($member, $context, $next_depth);
+      }
+
+      // If we only have one item then just return it as a single item.
+      // TODO: we may need to check cardinality of the field and be more
+      // strict about how we return the value.
+      if ($field_data['totalItems'] == 1){
+        return $members[0];
+      }
+      else {
+        return $members;
+      }
+    }
+    else {
+      $next_depth = $depth + 1;
+      return _tripal_update_remote_entity_field($field_data, $context, $next_depth);
+    }
+  }
+  // If this isn't depth 0 then we are rewriting keys as CV term accession.
+  else {
+
+    $value = array();
+    foreach ($field_data as $k => $v) {
+      // Skip the JSON-LD keys.
+      if ($k == '@id' or $k == '@type' or $k == '@context') {
+        continue;
+      }
+      // Find the term accession for this element, and if the key's value is an
+      // array then recurse.
+      $accession = $context[$k];
+      if (is_array($v)) {
+        $next_depth = $depth + 1;
+        $subvalue = _tripal_update_remote_entity_field($v, $context, $next_depth);
+        $value[$accession] = $value;
+      }
+      else {
+        $value[$accession] = $v;
+      }
+    }
+    return $value;
+  }
+}
+
+/**
+ * Behaves similar to the field_info_field() function but for remote fields.
+ *
+ * Returns a "fake" field info array for fields attached to content types
+ * on remote Tripal sites.
+ *
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @param $bundle_accession
+ *   The controlled vocabulary term accession for the content type
+ *   on the remote Tripal site.
+ * @param $field_accession
+ *   The controlled vocabulary term accession for the property (i.e. field) of
+ *   the Class (i.e. content type).
+ * @return
+ *   An array similar to that returned by the field_info_field function
+ *   of Drupal for local fields.
+ */
+function tripal_get_remote_field_info($site_id, $bundle_accession, $field_accession){
+
+  // Get the site documentation (loads from cache if already retrieved).
+  $site_doc = tripal_get_remote_API_doc($site_id);
+
+  // Get the property from the document for this field.
+  $property = tripal_get_remote_field_doc($site_id, $bundle_accession, $field_accession);
+
+  // Now create the fake field and instance.
+  list($vocab, $accession) = explode(':', $field_accession);
+  $field_name = 'tripal_remote_site_' . $site_id . '_' . $field_accession;
+
+  $field = array(
+    'field_name' => $field_name,
+    'type' => $field_name,
+    'storage' => array(
+      'type' => 'tripal_remote_site'
+    ),
+  );
+  return $field;
+}
+
+/**
+ * Behaves similar to the field_info_instance() function but for remote fields.
+ *
+ * Returns a "fake" instance info array for fields attached to content types
+ * on remote Tripal sites.
+ *
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @param $bundle_accession
+ *   The controlled vocabulary term accession for the content type
+ *   on the remote Tripal site.
+ * @param $field_accession
+ *   The controlled vocabulary term accession for the property (i.e. field) of
+ *   the Class (i.e. content type).
+ * @return
+ *   An array similar to that returned by the field_info_instance function
+ *   of Drupal for local fields.
+ */
+function tripal_get_remote_field_instance_info($site_id, $bundle_accession, $field_accession){
+
+  // Get the site documentation (loads from cache if already retrieved).
+  $site_doc = tripal_get_remote_API_doc($site_id);
+
+  // Get the property from the document for this field.
+  $property = tripal_get_remote_field_doc($site_id, $bundle_accession, $field_accession);
+
+  list($vocab, $accession) = explode(':', $field_accession);
+  $field_name = 'tripal_remote_site_' . $site_id . '_' . $field_accession;
+
+  list($vocab, $accession) = explode(':', $field_accession);
+  $instance = array(
+    'label' => $property['hydra:title'],
+    'description' => $property['hydra:description'],
+    'formatters' => $property['tripal_formatters'],
+    'settings' => array(
+      'term_vocabulary' => $vocab,
+      'term_accession' => $accession
+    ),
+    'field_name' => $field_name,
+    'entity_type' => 'TripalEntity',
+    'bundle_name' => $bundle_accession,
+  );
+  return $instance;
+}
+
+
+/**
+ * Retreive the bundle (content type) information from a remote Tripal site.
+ *
+ * The array returned is equivalent to the Hydra Vocabulary "supportedClass"
+ * stanza.
+ *
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @param $bundle_accession
+ *   The controlled vocabulary term accession for the content type
+ *   on the remote Tripal site.
+ * @return
+ *   A PHP array corresponding to the Hydra Class stanza (i.e. a content type).
+ *   Returns NULL if the class ID cannot be found.
+ */
+function tripal_get_remote_bundle_doc($site_id, $bundle_accession){
+
+  // Get the site documentation (loads from cache if already retrieved).
+  $site_doc = tripal_get_remote_API_doc($site_id);
+
+  // Get the class that matches this bundle.
+  $classes = $site_doc['supportedClass'];
+  $class = NULL;
+  foreach ($classes as $item) {
+    if ($item['@id'] == $bundle_accession) {
+      return $item;
+    }
+  }
+  return NULL;
+}
+
+/**
+ * Retrieves the field information for a content type from a remote Tripal site.
+ *
+ * The array returned is equivalent to the Hydra Vocabulary "supportedProperty"
+ * stanza that belongs to a Hydra Class (content type).
+ *
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @param $bundle_accession
+ *   The controlled vocabulary term accession for the content type
+ *   on the remote Tripal site.
+ * @param $field_accession
+ *   The controlled vocabulary term accession for the property (i.e. field) of
+ *   the Class (i.e. content type).
+ * @return
+ *   A PHP array corresponding to the Hydra property stanza (field) that
+ *   belongs to the given Class (i.e. a content type).  Retruns NULL if the
+ *   property cannot be found.
+ */
+function tripal_get_remote_field_doc($site_id, $bundle_accession, $field_accession){
+
+  // Get the site documentation (loads from cache if already retrieved).
+  $site_doc = tripal_get_remote_API_doc($site_id);
+
+  $class = tripal_get_remote_bundle_doc($site_id, $bundle_accession);
+  $properties = $class['supportedProperty'];
+  foreach ($properties as $item) {
+    if ($item['property'] == $field_accession) {
+      return $item;
+    }
+  }
+  return NULL;
+}
+
+
+/**
+ * Retrieves the list of download formatters for a remote field.
+ *
+ * All Tripal fields support the abilty for inclusion in files that can
+ * downloaded.  This function is used to identify what formatters these
+ * fields are appropriate for. If those download formatter classes exist
+ * on this site then the field can be used with that formatter.
+ *
+ * @param $site_id
+ *   The numeric site ID for the remote Tripal site.
+ * @param $bundle_accession
+ *   The controlled vocabulary term accession for the content type
+ *   on the remote Tripal site.
+ * @param $field_accession
+ *   The controlled vocabulary term accession for the property (i.e. field) of
+ *   the Class (i.e. content type).
+ * @return
+ *   An array of field downloader class names that are compoatible with the
+ *   field and which exist on this site.
+ */
+function tripal_get_remote_field_formatters($site_id, $bundle_accession, $field_accession) {
+
+  $flist = array();
+
+  // Get the site documentation (loads from cache if already retrieved).
+  $site_doc = tripal_get_remote_API_doc($site_id);
+  $property = tripal_get_remote_field_doc($site_id, $bundle_accession, $field_accession);
+  if (!$property) {
+    return $flist;
+  }
+
+  $formatters = $property['tripal_formatters'];
+  foreach($formatters as $formatter) {
+    if (tripal_load_include_downloader_class($formatter)) {
+      $flist[$formatter] = $formatter::$full_label;
+    }
+  }
+  return $flist;
 }

tripal_ws/includes/TripalFields/remote__data/remote__data.inc

@@ -205,45 +205,7 @@ class remote__data extends WebServicesField {
        list($ctype, $qdata) = explode('?', $query);
      }
 
-     // Build the URL to the remote web services.
-     $ws_version = $this->remote_site->version;
-     $ws_url = $this->remote_site->url;
-     $ws_url = trim($ws_url, '/');
-     $ws_url .= '/web-services/content/' . $ws_version . '/' . $ctype;
-
-     // Build the Query and make and substitions needed.
-     //dpm($ws_url . '?' . $query);
-     $options = array(
-       'data' => $qdata,
-     );
-     $data = drupal_http_request($ws_url, $options);
-
-     if (!$data) {
-       tripal_report_error('tripal_ws', TRIPAL_ERROR,
-           t('Could not connect to the remote web service.'));
-       return FALSE;
-     }
-
-     // If the data object has an error then this is some sort of
-     // connection error (not a Tripal web servcies error).
-     if (property_exists($data, 'error')) {
-       tripal_report_error('tripal_ws', TRIPAL_ERROR,
-           'Web Services error on remote site: %error.',
-           array('%error' => $data->error));
-       return FALSE;
-     }
-
-     // We got a response, so convert it to a PHP array.
-     $data = drupal_json_decode($data->data);
-
-     // Check if there was a Tripal Web Services error.
-     if (array_key_exists('error', $data)) {
-       $error = '</pre>' . print_r($data['error'], TRUE) . '</pre>';
-       tripal_report_error('tripal_ws', TRIPAL_ERROR,
-           'Web Services error on remote site: %error.',
-           array('%error' => $error));
-       return FALSE;
-     }
+     $data = tripal_query_remote_site($this->remote_site->site_id, $query);
 
      return $data;
    }