Fixed merge conflict

tripal/api/tripal.entities.api.inc

@@ -725,7 +725,7 @@ function tripal_create_bundle_fields($bundle, $term) {
     }
     
     // Make sure the term is not used for any other existing field instance.
-    $skip = TRUE;
+    $skip = FALSE;
     foreach ($existing_instances as $existing_name => $existing_instance) {
       // If this instance term is the same as this exsiting term and the 
       // instance name is not the same then we have a problem.

tripal/api/tripal.quotas.api.inc

@@ -70,22 +70,14 @@ function tripal_remove_user_quota($uid) {
 function tripal_get_user_usage($uid) {
   // Get the user's current file usage
   $sql = "
-    SELECT DISTINCT FU.fid
-    FROM {file_usage} FU
-      INNER JOIN {file_managed} FM ON FM.fid = FU.fid and FU.module = 'tripal'
-    WHERE FM.uid = :uid
+    SELECT SUM(filesize) FROM (
+    SELECT DISTINCT FM.fid, FM.filename, FM.filesize
+      FROM file_usage FU
+        INNER JOIN file_managed FM ON FM.fid = FU.fid and FU.module = 'tripal'
+      WHERE FM.uid = :uid) AS foo
   ";
-  $fids = db_query($sql, [
-    ':uid' => $uid
-  ]);
-  $total_size = 0;
-  while ($fid = $fids->fetchObject()) {
-    $sql = "SELECT filesize FROM {file_managed} WHERE fid = :fid";
-    $total_size += db_query($sql, [
-      ':fid' => $fid->fid
-    ])->fetchObject()->filesize;
-  }
-  return $total_size;
+  $total_usage = db_query($sql, [':uid' => $uid])->fetchField();
+  return ($total_usage);
 }
 
 /**

tripal/api/tripal.terms.api.inc

@@ -378,6 +378,7 @@ function tripal_get_term_details($vocabulary, $accession) {
 
   if (empty($vocabulary) OR empty($accession)) {
     tripal_report_error('tripal_term', TRIPAL_ERROR, "Unable to retrieve details for term due to missing vocabulary and/or accession");
+    return FALSE;
   }
 
   // TODO: we need some sort of administrative interface that lets the user
@@ -390,7 +391,7 @@ function tripal_get_term_details($vocabulary, $accession) {
     $function = $module . '_vocab_get_term';
     if (function_exists($function)) {
       $term = $function($vocabulary, $accession);
-      
+
       // Make sure the term has a URL. If it does not, then use the Tripal
       // interface as the URL for the term.
       $url_missing = FALSE;
@@ -795,7 +796,8 @@ function tripal_get_term_lookup_form_result($form, $form_state, $field_name = ''
   $values = array();
   $selected = array();
   if ($field_name) {
-    if (array_key_exists('term_match', $form_state['values'][$field_name]['und'][$delta])) {
+    if (array_key_exists('term_match', $form_state['values'][$field_name]['und'][$delta]) and 
+      array_key_exists('terms_list', $form_state['values'][$field_name]['und'][$delta]['term_match'])) {
       $values = $form_state['values'][$field_name]['und'][$delta]['term_match']['terms_list'];
     }
   }

tripal/includes/TripalEntityController.inc

@@ -112,9 +112,9 @@ class TripalEntityController extends EntityAPIController {
     // If no title was supplied then we should try to generate one using the
     // default format set by admins.
     if (!$title) {     
-      $title = tripal_get_title_format($bundle_entity);
+      $title = tripal_get_title_format($bundle);
     }
-    $title = tripal_replace_entity_tokens($title, $entity, $bundle_entity);
+    $title = tripal_replace_entity_tokens($title, $entity, $bundle);
     
     if ($title) {
       db_update('tripal_entity')

tripal/includes/TripalImporter.inc

@@ -113,6 +113,38 @@ class TripalImporter {
    * value is empty then the path will be the default.
    */
   public static $menu_path = '';
+  
+  
+  /**
+   * If your importer requires more flexibility and advanced features than
+   * the TripalImporter provides, you can indicate a callback function. If set,
+   * the callback will be used to provide the importer interface to the
+   * end-user.  However, because this bypasses the class infrastructure the
+   * run() function will also not be available and your importer must be
+   * fully self-sufficient outside of this class.  The benefit for using a
+   * TripalImporter despite your loader being self-sufficient is that Tripal
+   * will treat your loader like all others providing a consistent location
+   * in the menu and set of permissions.
+   * 
+   * Note: use of a callback is discouraged as the importer provides a 
+   * consistent workflow for all importers.  Try your best to fit your importer
+   * within the class.  Use this if you absolutely cannot fit your importer
+   * into  TripalImporter implementation.
+   * 
+   */
+  public static $callback = '';
+  
+  /**
+   * The name of the module that provides the callback function.
+   */
+  public static $callback_module = '';
+  
+  /**
+   * An include path for the callback function.  Use a relative path within
+   * this scope of this module
+   * (e.g. includes/loaders/tripal_chado_pub_importers).
+   */
+  public static $callback_path = '';
 
   // --------------------------------------------------------------------------
   //                  PRIVATE MEMBERS -- DO NOT EDIT or OVERRIDE

tripal/includes/tripal.extensions.inc

@@ -1,780 +0,0 @@
-<?php
-
-function tripal_extensions_form($form, &$form_state = NULL) {
-
-  // Get the RSS feed XML from the tripal.info website and save it to
-  // a temp file so that we don't have to keep pulling the XML
-  // everythime the page is loaded. If the temp file is older than 1 hours
-  // then we'll pull it again. The number of seconds in an hour is 3600.
-  $tmp_file = sys_get_temp_dir() . '/tripal_rss_extensions.xml';
-  if (!file_exists($tmp_file) or time() - filemtime($tmp_file) > 3600) {
-    $content = file_get_contents("http://tripal.info/rss/extensions.xml");
-    file_put_contents($tmp_file, $content);
-  }
-  else {
-    $content = file_get_contents($tmp_file);
-  }
-  $xml = new SimpleXmlElement($content);
-  $namespace = "http://tripal.info/rss/extensions/";
-
-
-  $tab = array_key_exists('tab', $_GET) ? $_GET['tab'] : '';
-
-  // Get any filters by categories.
-  $filters = array();
-  if (array_key_exists('values', $form_state)) {
-    foreach ($form_state['values'] as $key => $value) {
-      // Get the category to be filtered on.
-      $matches = array();
-      if (preg_match('/^categories-(.*?)$/', $key, $matches)) {
-        if ($value == 'any') {
-          continue;
-        }
-        $filters[$matches[1]] = $value;
-      }
-    }
-  }
-
-  // Parse the items into an array indexed by type and compatible versions.
-  $extensions = array();
-  $types = array();
-  $type_ids = array();
-  $categories = array();
-  $cvs = array();
-  $tvs = array();
-  foreach ($xml->channel->item as $extension) {
-    // Get the type of extension, convert that into a machine-readable name,
-    // and build the array of types.
-    $type = (string) $extension->category;
-    $type_id = preg_replace('/[^\w]/','_', strtolower($type));
-    $type_ids[$type] = $type_id;
-    if (!in_array($type_id, $types)) {
-      $types[$type] = 1;
-    }
-
-    // Get the categories list for this item.
-    $cats = preg_split('/, /', (string) $extension->children($namespace)->categories);
-
-    // Get the guid for this extension
-    $guid = (string) $extension->guid;
-
-    // In order to get fields in the 'tripal_extension' name space we must
-    // pass in the $namespace to the children function.  We first get the
-    // Tripal versions, then the chado versions and organize the elements
-    // accordintly.
-    $tvs_temp = preg_split('/, /', (string) $extension->children($namespace)->tripal_version);
-    foreach($tvs_temp as $tv) {
-      $tvs[$tv] = 1;
-      $cvs_temp = preg_split('/, /', (string) $extension->children($namespace)->chado_version);
-      foreach($cvs_temp as $cv) {
-        $cvs[$cv] = 1;
-
-        // Keep track of the categories this item has been assigned.
-        foreach ($cats as $cat) {
-          $categories[$tv][$cv][$type][$cat] = 1;
-          $categories['any'][$cv][$type][$cat] = 1;
-          $categories[$tv]['any'][$type][$cat] = 1;
-          $categories['any']['any'][$type][$cat] = 1;
-        }
-
-        // If there are filters then only include extensions that match the filters.
-        if (array_key_exists($type_id, $filters) and !in_array($filters[$type_id], $cats)) {
-          continue;
-        }
-
-        // Index the items by type, tripal version and chado version.
-        $item = array();
-        foreach ($extension->children() as $child) {
-          $item[$child->getName()] = (string) $child;
-        }
-        foreach ($extension->children($namespace) as $child) {
-          $item[$namespace][$child->getName()] = (string) $child;
-        }
-        $extensions[$tv][$cv][$type][$guid] = $item;
-        $extensions['any'][$cv][$type][$guid] = $item;
-        $extensions[$tv]['any'][$type][$guid] = $item;
-        $extensions['any']['any'][$type][$guid] = $item;
-      }
-    }
-  }
-
-  // Convert the arrays from an associative array into a normal array, and sort.
-  $types = array_keys($types);
-  sort($types);
-  $cvs = array_keys($cvs);
-  sort($cvs);
-  $tvs = array_keys($tvs);
-  sort($tvs);
-
-  // Get the Chado version and convert to the expected format
-  $chado_version = chado_get_version(TRUE);
-  $chado_version = preg_replace('/^(\d\.\d).*$/', "v$1x", $chado_version);
-  $my_chado_version = $chado_version;
-  // The default value can come from the pager links (thus via $_GET) or
-  // via ajax call (thus via $form_state).
-  if (array_key_exists('cv', $_GET)) {
-    $chado_version = $_GET['cv'];
-  }
-  if (array_key_exists('values', $form_state) and array_key_exists('cv', $form_state['values'])) {
-    $chado_version = $form_state['values']['cv'];
-  }
-
-  // Get the Tripal version. This is the version set in the tripal.info
-  $info = system_get_info('module', 'tripal');
-  $tripal_version = $info['version'];
-  $tripal_version = preg_replace('/^.*?-(\d\.\d+).*$/', "v$1", $tripal_version);
-  $my_tripal_version = $tripal_version;
-  if (array_key_exists('tv', $_GET)) {
-    $tripal_version = $_GET['tv'];
-  }
-  if (array_key_exists('values', $form_state) and array_key_exists('tv', $form_state['values'])) {
-    $tripal_version = $form_state['values']['tv'];
-  }
-
-  // Add the instructions.
-  $form['instructions'] = array(
-    '#type' => 'item',
-    '#markup' => t('This page will help you find extensions that are available
-      for Tripal.  Select an extension type from the vertical tabs. The content
-      of this page is constructed from an RSS feed provided by tripal.info.
-      There may be no content if the tripal.info site is unavailable. The RSS
-      feed will be cached for one hour.')
-  );
-  // Add the filters fieldset.
-  $form['filters'] = array(
-    '#type' => 'fieldset',
-    '#title' => 'Filters',
-    '#description' => t('You can filter which extensions are visible by
-      changing the Tripal ahd Chado versions. By default only those
-      extensions that are compatible with the currently installed Tripal
-      and Chado verions are shown.'),
-    '#collapsible' => TRUE,
-    '#collapsed' => TRUE,
-  );
-  $summary_message = 'Currently showing extensions compatible with ';
-  if ($tripal_version != 'any' and $chado_version != 'any') {
-    $summary_message .= "<strong>Tripal $tripal_version</strong> and <strong>Chado $chado_version</strong>.";
-  }
-  elseif ($tripal_version == 'any' and $chado_version != 'any') {
-    $summary_message .= "<strong>any Tripal</strong> version and <strong>Chado $chado_version</strong>.";
-  }
-  elseif ($tripal_version != 'any' and $chado_version == 'any') {
-    $summary_message .= "<strong>Tripal $tripal_version</strong> and <strong>any Chado</strong> version.";
-  }
-  elseif ($tripal_version == 'any' and $chado_version == 'any') {
-    $summary_message .= "<strong>any Tripal</strong> version and <strong>any Chado</strong> version.";
-  }
-  $form['filter_summary'] = array(
-    '#type' => 'item',
-    '#markup' => $summary_message,
-  );
-
-  // Add the Tripal version select box.
-  $options = array();
-  $options['any'] = '--Any--';
-  foreach ($tvs as $tv) {
-    $options[$tv] = $tv;
-  }
-  $form['filters']['tv'] = array(
-    '#type' => 'select',
-    '#title' => 'Tripal',
-    '#options' => $options,
-    '#default_value' => $tripal_version,
-    '#ajax'  => array(
-      'callback' => "tripal_extensions_form_ajax_callback",
-      'wrapper'  => 'tripal_extensions',
-      'effect'   => 'fade',
-      'method'   => 'replace',
-    ),
-    '#prefix' => '<div style="float: left;">',
-    '#suffix' => '</div>'
-  );
-
-  // Add the Chado version select box.
-  $options = array();
-  $options['any'] = '--Any--';
-  foreach ($cvs as $cv) {
-    $options[$cv] = $cv;
-  }
-  $form['filters']['cv'] = array(
-    '#type' => 'select',
-    '#title' => 'Chado',
-    '#options' => $options,
-    '#default_value' => $chado_version,
-    '#ajax'  => array(
-      'callback' => "tripal_extensions_form_ajax_callback",
-      'wrapper'  => 'tripal_extensions',
-      'effect'   => 'fade',
-      'method'   => 'replace',
-    ),
-    '#prefix' => '<div style="float: left; padding-left: 10px">',
-    '#suffix' => '</div>'
-  );
-
-  // Add the vertical tabs
-  $form['extensions'] = array(
-    '#type' => 'vertical_tabs',
-    '#default_tab' => $tab,
-  );
-
-  // Add the fieldsets for each type
-  foreach ($types as $type) {
-    $form[$type] = array(
-      '#id'          => $type_ids[$type],
-      '#type'        => 'fieldset',
-      '#title'       => $type . 's',
-      '#collapsed'   => TRUE,
-      '#collapsible' => TRUE,
-      '#group'       => 'extensions',
-    );
-  }
-
-  // Iterate through all of the extensions and add them to the form.
-  $compatible_extensions = array();
-  if (array_key_exists($tripal_version, $extensions) and
-      array_key_exists($chado_version, $extensions[$tripal_version])) {
-    $compatible_extensions = $extensions[$tripal_version][$chado_version];
-  }
-  tripal_extension_form_add_extensions($form, $form_state,
-    $compatible_extensions, $categories, $tripal_version,
-    $chado_version, $my_tripal_version, $my_chado_version, $type_ids,
-    $namespace, $filters);
-
-  foreach ($types as $type) {
-    if (count(element_children($form[$type])) == 0) {
-      $form[$type]['empty'] = array(
-        '#type' => 'item',
-        '#markup' => '<strong>There are no matching ' . strtolower($type) . '(s).</strong>',
-      );
-    }
-  }
-
-  $form['#prefix'] = '<div id="tripal_extensions">';
-  $form['#suffix'] = '</div>';
-  $form['#submit'][] = 'tripal_extensions_form_submit';
-  return $form;
-}
-
-/**
- * Adds each extension to the form.
- *
- * This function exits to simplify the the tripal_extension_form()
- * function.
- */
-function tripal_extension_form_add_extensions(&$form, $form_state, $extensions,
-  $categories, $tripal_version, $chado_version, $my_tripal_version,
-  $my_chado_version, $type_ids, $namespace, $filters) {
-
-  // Iterate through the extensions. We will add a pager for
-  // each type of extension, and display only those that should appear
-  // on the page.
-  $type_index = 0;
-  foreach ($extensions as $type => $extensions) {
-    $total_items = count($extensions);
-
-    // Indicate how many matching extensions were found
-    $form[$type]['total_found'] = array(
-      '#type' => 'item',
-      '#markup' => '<strong>Found ' . $total_items . ' matching ' . strtolower($type) . '(s)</strong>',
-    );
-
-    // Add the category select box;
-    $cats = array_keys($categories[$tripal_version][$chado_version][$type]);
-    sort($cats);
-    $options = array();
-    $options['any'] = '--Any--';
-    foreach ($cats as $cat) {
-      $options[$cat] = $cat;
-    }
-    // The default value can come from the pager links (thus via $_GET) or
-    // via ajax call (thus via $form_state).
-    $default_filter = '';
-    if (array_key_exists('categories-' . $type_ids[$type], $_GET)) {
-      $default_filter = $_GET['categories-' . $type_ids[$type]];
-    }
-    if (array_key_exists('values', $form_state) and array_key_exists('categories-' . $type_ids[$type], $form_state['values'])) {
-      $default_filter = $form_state['values']['categories-' . $type_ids[$type]];
-    }
-    $form[$type]['filters']['categories-' . $type_ids[$type]] = array(
-      '#type' => 'select',
-      '#title' => 'Filter by Category',
-      '#options' => $options,
-      '#default_value' => $default_filter,
-      '#ajax'  => array(
-        'callback' => "tripal_extensions_form_ajax_callback",
-        'wrapper'  => 'tripal_extensions',
-        'effect'   => 'fade',
-        'method'   => 'replace',
-      ),
-    );
-
-    // Initialize pager and gets current page
-    $num_per_page = 5;
-    $page = pager_default_initialize($total_items, $num_per_page, $type_index);
-
-    // Gets first record and last record to show
-    $start = ($page) * $num_per_page;
-    $end = ($start + $num_per_page < $total_items)? $start + $num_per_page : $total_items;
-    // Iterate through each of the elements and add them to the form if
-    // they are within the page
-    $extension_index = 0;
-    foreach ($extensions as $guid => $extension) {
-      // Skip items that aren't on our current page.
-      if ($extension_index < $start or $extension_index >= $end) {
-        $extension_index++;
-        continue;
-      }
-
-      $extension['title'] = trim($extension['title']);
-
-      // If this is an extension module then there will be a home page for it
-      $home_page = '';
-      if (array_key_exists('home_page', $extension[$namespace])) {
-        $home_page = "<strong>Project Home: </strong>" . $extension[$namespace]['home_page'] . "</br>";
-      }
-
-      // Determine if this extension is compatible with this site.
-      $incompatible = '';
-      $tvs_temp = preg_split('/, /', $extension[$namespace]['tripal_version']);
-      $cvs_temp = preg_split('/, /', $extension[$namespace]['chado_version']);
-      if (!in_array($my_tripal_version, $tvs_temp)) {
-        $incompatible .= "<li>This extension is not compatible with this version of Tripal.</li>";
-      }
-      if (!in_array($my_chado_version, $cvs_temp)) {
-        $incompatible .= "<li>This extension is not compatible with the installed Chado version.</li>";
-      }
-      $incompatible = t($incompatible);
-
-      // Determine if this extension is already installed.
-      $is_installed = '';
-      switch ($type) {
-        case 'Bulk Loader Template':
-          if (module_exists('tripal_bulk_loader')) {
-            $blk_id = db_select('tripal_bulk_loader_template' ,'tblt')
-              ->fields('tblt', array('template_id'))
-              ->condition('name', $extension['title'])
-              ->execute()
-              ->fetchField();
-            if ($blk_id) {
-              $is_installed = '<li>A bulk loader template with this name is already installed.</li>';
-            }
-          }
-          break;
-        case 'Materialized View':
-          $mview_id = chado_get_mview_id($extension[$namespace]['mview_name']);
-          if ($mview_id) {
-            $is_installed = '<li>A materialized view with this name is already installed.</li>';
-          }
-          break;
-        case 'Extension Module':
-          if (array_key_exists('drupal_project', $extension[$namespace]) and
-              module_exists($extension[$namespace]['drupal_project'])) {
-            $is_installed = '<li>A module with this name is already installed.</li>';
-          }
-          break;
-        default:
-          break;
-      }
-      $is_installed = t($is_installed);
-
-      // Does this module appear to be available on Drupal.org?
-      $project = '';
-      if ($type == 'Extension Module') {
-        // Does it have a drupal project name?
-        if (!array_key_exists('drupal_project', $extension[$namespace])) {
-          // Since it doesn't have a drupal project name is it in a sandbox?
-          if (!preg_match('/www.drupal.org\/sandbox/', $extension[$namespace]['home_page'])) {
-            $project = t("<li>This module does not appear to be available as a " .
-              "full Drupal project and thus cannot ".
-              "be downloaded here. You may have to manually download it.</li>");
-          }
-          else {
-            $project = ("<li>This module is in a sandbox on Drupal.org, and
-              cannot be downloaded from here. You will have to manually download
-              this module.</li>");
-          }
-        }
-      }
-
-      // Make sure the bulk loader module is installed, or we cannot provide
-      // the bulk loader import button.
-      $other = '';
-      if ($type == 'Bulk Loader Template' and !module_exists('tripal_bulk_loader')) {
-        $other = t('<li>The bulk loader
-          module is not enabled. If you would like to import a loading template
-          please enable it.</li>');
-      }
-
-      // If the user click's the button to import the extension then we
-      // need the item in the submit function so we can process the import.
-      $form[$type]['extension-' . $guid] = array(
-        '#type' => 'value',
-        '#value' => $extension,
-      );
-
-      $notices = '';
-      if ($is_installed) {
-        $notices = '<div class="messages status"><ul>' . $is_installed . '</ul></div>';
-      }
-      $warnings = '';
-      if ($project or $other) {
-        $warnings = '<div class="messages warning"><ul>' .
-          $project .  ' ' .
-          $other . '</ul></div>';
-      }
-      $errors = '';
-      if ($incompatible) {
-        $errors = '<div class="messages error"><ul>' . $incompatible .  '</ul></div>';
-      }
-
-      $state = '';
-      if (array_key_exists('dev_stage', $extension[$namespace])) {
-        $state = '<strong>Development State: </strong>' . $extension[$namespace]['dev_stage'] . "</br>";
-      }
-
-      // Create the form elements that we'll later theme into tables.
-      $form[$type][$guid]['header'] = array(
-        '#markup' => l($extension['title'], $extension['link']),
-      );
-      $form[$type][$guid]['details'] = array(
-        '#markup' => "" .
-        "<strong>Type:</strong> " . $type . "</br>" .
-        "<strong>Categories: </strong>" . $extension[$namespace]['categories'] . "</br>" .
-        "<strong>Authors: </strong>" . $extension[$namespace]['authors'] . "</br>" .
-        $state .
-        "<strong>Chado compatible versions: </strong>" . $extension[$namespace]['chado_version'] . "</br>" .
-        "<strong>Tripal compatible versions: </strong>" . $extension[$namespace]['tripal_version'] . "</br>" .
-        $home_page .
-        "<strong>tripal.info Page: </strong>" . l($extension['link'], $extension['link']) . "</br>" .
-        $notices .
-        $warnings .
-        $errors .
-        "<p>" . $extension['description'] . "</p>",
-      );
-      // Add an import button to each of types that can support import.
-      switch ($type) {
-        case 'Bulk Loader Template':
-          if (!$incompatible and !$is_installed and !$project) {
-            $form[$type][$guid]['import'] = array(
-              '#type' => 'submit',
-              '#value' => "Import Loader",
-              '#name' => "import-" . $guid,
-            );
-          }
-          break;
-        case 'Materialized View':
-          if (!$incompatible and !$is_installed and !$project) {
-            $form[$type][$guid]['import'] = array(
-              '#type' => 'submit',
-              '#value' => "Import MView",
-              '#name' => "import-" . $guid,
-            );
-          }
-          break;
-        case 'Extension Module':
-          if (!$incompatible and !$is_installed and !$project) {
-            $form[$type][$guid]['import'] = array(
-              '#type' => 'submit',
-              '#value' => "Download Module",
-              '#name' => "import-" . $guid,
-            );
-          }
-          break;
-        default:
-          break;
-      }
-      $form[$type][$guid]['#theme'] = 'tripal_extensions_form_tables';
-      $extension_index++;
-    }
-
-    // Now create and theme the pager.
-    $pager = array(
-      'tags' => array(),
-      'element' => $type_index,
-      'parameters' => array(
-        'tab' => $type_ids[$type],
-        'cv' => $chado_version,
-        'tv' => $tripal_version,
-      ),
-      'quantity' => $num_per_page,
-    );
-
-    // now add the category filters to the params array
-    foreach ($filters as $filter_id => $value) {
-      $pager['parameters']['categories-' . $filter_id] = $value;
-    }
-
-    // because this may be an ajax callback, the theme_pager will set the URL to be
-    // "system/ajax", so we need to reset that
-    $pager = theme('pager', $pager);
-    global $base_path;
-    $pager = str_replace($base_path . "system/ajax", "", $pager) ;
-
-    $form[$type]['pager'] = array(
-      '#type' => 'item',
-      '#markup' => $pager,
-    );
-    $type_index++;
-  }
-}
-
-/**
- * Process the import buttons.
- *
- * @param $form
- * @param $form_state
- */
-function tripal_extensions_form_submit($form, &$form_state) {
-  // get the guid
-  $clicked_button = $form_state['clicked_button']['#name'];
-  $guid = preg_replace('/^import-(\d+)$/', "$1", $clicked_button);
-  if ($guid) {
-    $namespace = "http://tripal.info/rss/extensions/";
-    $extension = $form_state['values']['extension-' . $guid];
-    $type = $extension['category'];
-    switch ($type) {
-      case 'Bulk Loader Template':
-        $options = array(
-          'template_name' => $extension['title'],
-          'template_array' => $extension[$namespace]['bulkldr_export'],
-          'strict' => TRUE,
-        );
-        $errors = array();
-        $warnings = array();
-        $success = tripal_insert_bulk_loader_template($options, $errors, $warnings);
-        if ($success) {
-          drupal_set_message("Bulk loader succesfully added.");
-        }
-        else {
-          drupal_set_message("Error importing this bulk loader.",  'error');
-          if (count($errors) > 0) {
-            foreach($errors as $field => $message) {
-              drupal_set_message($message, 'error');
-            }
-          }
-        }
-        break;
-      case 'Materialized View':
-        $module_name = 'tripal';
-        $mview_name = $extension[$namespace]['mview_name'];
-        $mview_schema = $extension[$namespace]['mview_schema'];
-        $mview_sql = $extension[$namespace]['mview_sql'];
-
-        // Validate the contents of the schema array.
-        // TODO: security issue!! Before using 'eval' on this string
-        // we should make sure the array is valid and there is no hidden
-        // PHP code.
-        $schema_array = array();
-        $success = eval("\$schema_array = $mview_schema;");
-        $error = chado_validate_custom_table_schema($schema_array);
-        if ($error) {
-          drupal_set_message("Cannot import Materialized View: $error", "error");
-        }
-        chado_add_mview($mview_name, $module_name, $schema_array, $mview_sql);
-        break;
-      case 'Extension Module':
-        if (array_key_exists('drupal_project', $extension[$namespace])) {
-          module_load_include('module', 'update', 'update');
-          module_load_include('inc', 'update', 'update.manager');
-          $project = $extension[$namespace]['drupal_project'];
-          $tar = tripal_extensions_get_latest_module_version($project);
-          if (!$tar) {
-            drupal_set_message('Cannot find a suitable release of this module
-              for download. You may need to manually install this module.', 'error');
-          }
-          else {
-
-            // Download the file from the Drupal repository
-            $local_cache = update_manager_file_get($tar);
-            if (!$local_cache) {
-              drupal_set_message('Cannot download the file. Check the
-                "Recent log messages" for relavent errors.', 'error');
-            }
-            else {
-              // The following code was borrowed from the update_manager_install_form_submit()
-              // of drupal in the modules/update/update.manager.inc file.
-              $directory = _update_manager_extract_directory();
-              try {
-                $archive = update_manager_archive_extract($local_cache, $directory);
-              }
-              catch (Exception $e) {
-                drupal_set_message('Cannot extract the file. Please check
-                  permissions in the modules directory', 'error');
-                return;
-              }
-              $archive_errors = update_manager_archive_verify($project, $local_cache, $directory);
-              if (!empty($archive_errors)) {
-                foreach ($archive_errors as $error) {
-                  drupal_set_message($error, 'error');
-                }
-              }
-              $project_location = $directory . '/' . $project;
-              try {
-                $updater = Updater::factory($project_location);
-              }
-              catch (Exception $e) {
-                drupal_set_message($e->getMessage(), 'error');
-                return;
-              }
-              $project_real_location = drupal_realpath($project_location);
-              $arguments = array(
-                'project' => $project,
-                'updater_name' => get_class($updater),
-                'local_url' => $project_real_location,
-              );
-              module_load_include('inc', 'update', 'update.authorize');
-              $filetransfer = new FileTransferLocal(DRUPAL_ROOT);
-              call_user_func_array('update_authorize_run_install', array_merge(array($filetransfer), $arguments));
-
-              drupal_set_message('It appears the module was downloaded and
-                extracted. You can now ' . l('enable this module'. 'admin/modules') . '.');
-            }
-          }
-        }
-        else {
-          drupal_set_message('Cannot download this module.  The Drpual project
-            name is not set. You may have to manually download this module, and
-            if possible encourage the developers to set the project name if it
-            has been fully published on Drupal.org already.', 'error');
-        }
-        break;
-      default:
-        break;
-    }
-  }
-}
-
-/**
- * The theme function for rendering each element's table.
- *
- * @param $variables
- */
-function theme_tripal_extensions_form_tables($variables) {
-   $element = $variables['element'];
-   $headers = array(
-     array(
-       'data' => drupal_render($element['header']),
-       'colspan' => 2,
-     )
-   );
-   $button = array_key_exists('import', $element) ? drupal_render($element['import']) : '&nbsp;';
-   $rows = array(
-     array(
-       array(
-         'data' => drupal_render($element['details']),
-       ),
-       array(
-         'data' => $button,
-         'width' => '5%',
-         'align' => 'right',
-       ),
-     ),
-   );
-
-   $table = array(
-     'header' => $headers,
-     'rows' => $rows,
-     'attributes' => array(),
-     'sticky' => FALSE,
-     'caption' => '',
-     'colgroups' => array(),
-     'empty' => '',
-   );
-
-  return theme_table($table);
-}
-
-/**
- * A callback function for the form
- *
- * @return unknown
- */
-function tripal_extensions_form_ajax_callback($form, $form_state) {
-  // This function need not do anything as the form will take care of
-  // updates needed. We just need to return the form.
-  return $form;
-}
-
-/**
- * Determines the most recent downloadable package for a module.
- *
- * This function will connect to Drupal.org's RSS feed for a project and
- * determine the most recent version of the module and return the URL
- * for it's package.
- *
- * @param $project
- */
-function tripal_extensions_get_latest_module_version($project_name) {
-  // First use the Drupal RESTful API to get the project
-  $url = "https://www.drupal.org/api-d7/node.json?type=project_module&field_project_machine_name=$project_name";
-  $result = json_decode(file_get_contents($url), TRUE);
-  $project = $result['list'][0];
-  $nid = $project['nid'];
-
-  // Second get the releases for this node and find releases for this Drupal
-  $url = "https://www.drupal.org/api-d7/node.json?type=project_release&field_release_project=$nid";
-  $result = json_decode(file_get_contents($url), TRUE);
-  $releases = $result['list'];
-  $drupal_version = VERSION;
-  $drupal_major = preg_replace('/^(\d+)\.\d+$/', "$1", $drupal_version);
-  $best_release = NULL;
-  foreach ($releases as $release) {
-    // This release must match the Drupal major version or it won't
-    // be compatiable. If it doesn't we'll skip it.
-    $release_version = $release['field_release_version'];
-    $release_drupal = preg_replace('/^(\d+)\.x-.*$/', "$1", $release_version);
-    if ($release_drupal != $drupal_major) {
-      continue;
-    }
-
-    // Set the current release to be the best one. On successive iterations
-    // we'll check to see if that is still true.
-    if ($best_release == NULL) {
-      $best_release = $release;
-      continue;
-    }
-    if ($release['field_release_version_major'] > $best_release['field_release_version_major']) {
-      $best_release = $release;
-      continue;
-    }
-    if ($release['field_release_version_patch'] > $best_release['field_release_version_patch']) {
-      $best_release = $release;
-      continue;
-    }
-
-    // If the best version has no extra part then let's keep it as this is the
-    // most stable release.
-    if (!$best_release['field_release_version_extra']) {
-      continue;
-    }
-
-    // Convert the 'extra' part to a numeric value that will let us compare
-    // the ranks of the release versions.
-    $extra_rank = array(
-      'unstable' => 1,
-      'alpha' => 2,
-      'beta' => 3,
-      'rc' => 4,
-    );
-    $matches = array();
-    $this_extra = 0;
-    if (preg_match('/^(.*?)(\d+)$/', $release['field_release_version_extra'], $matches)) {
-      $this_extra = $extra_rank[$matches[1]] . "." . $matches[2];
-    }
-    $best_extra = 0;
-    if (preg_match('/^(.*?)(\d+)$/', $best_release['field_release_version_extra'], $matches)) {
-      $best_extra = $extra_rank[$matches[1]] . "." . $matches[2];
-    }
-    if ($this_extra > $best_extra) {
-      $best_release = $release;
-      continue;
-    }
-  }
-  // If we have a best result then build the download string.
-  // TODO: we may need to make another web services call to get the actual
-  // download path, but for now we'll hard code the construction of it.
-  if ($best_release) {
-    return 'http://ftp.drupal.org/files/projects/' . $project_name . '-' . $best_release['field_release_version'] . '.tar.gz';
-  }
-  return '';
-}

tripal/includes/tripal.user.inc

@@ -21,7 +21,6 @@
        INNER JOIN {file_usage} FU on FM.fid = FU.fid and FM.uid = :user_id
        LEFT JOIN {tripal_expiration_files} TGEF on TGEF.fid = FU.fid
      WHERE FU.module = 'tripal'
-     GROUP BY FM.fid, TGEF.expiration_date
      ORDER BY FM.filename
    ";
    $files = db_query($sql, array(':user_id' => $uid));

tripal/tripal.module

@@ -131,35 +131,6 @@ function tripal_menu() {
     'access arguments' => array('administer tripal'),
   );
 
-  // Menu items for facilitating import of extension modules.
-  $items['admin/tripal/extension/available'] = array(
-    'title' => 'Available Extensions',
-    'description' => t('Look for extensions to add new functionality to this
-        site. Tripal can be extended with new functionality developed
-        by other Tripal site developers. These include modules with new or
-        different functionality, bulk loading templates, or materialized
-        views.  Anyone can create new extensions and share those for
-        others to use.  Once shared they will appear in this list.'),
-    'access arguments' => array('administer tripal'),
-    'page callback' => 'drupal_get_form',
-    'page arguments' => array('tripal_extensions_form'),
-    'type' => MENU_NORMAL_ITEM,
-    'file' => 'includes/tripal.extensions.inc',
-    'file path' => drupal_get_path('module', 'tripal'),
-    'weight' => -100
-  );
-//   $items['admin/tripal/extension/import'] = array(
-//     'title' => 'Import Extensions',
-//     'description' => 'Provides a list of the available extensions that are registered at the tripal.info site. From this page you can easily import or install extensions to your site.',
-//     'page callback' => 'drupal_get_form',
-//     'page arguments' => array('tripal_extensions_form'),
-//     'access arguments' => array('administer tripal'),
-//     'type' => MENU_NORMAL_ITEM,
-//     'file' =>  'includes/tripal.extensions.inc',
-//     'file path' => drupal_get_path('module', 'tripal'),
-//     'weight' => -100,
-//   );
-
   /**
    * Jobs Management
    */
@@ -334,11 +305,13 @@ function tripal_menu() {
     'type' => MENU_CALLBACK,
   );
 
-
+  /*
+   * Data Loaders (TripalImporter)
+   */
   $items['admin/tripal/loaders'] = array(
     'title' => 'Data Loaders',
     'description' => t('Tools facilitating data import.'),
-    'access arguments' => array('load tripal data'),
+    'access arguments' => array('administer tripal'),
     'type' => MENU_NORMAL_ITEM,
     'weight' => 6
   );
@@ -348,19 +321,36 @@ function tripal_menu() {
   foreach ($importers as $class_name) {
     tripal_load_include_importer_class($class_name);
     if (class_exists($class_name)) {
-      $menu_path = 'admin/tripal/loaders/' . $class_name::$machine_name;
+      $machine_name = $class_name::$machine_name;
+      $menu_path = 'admin/tripal/loaders/' . $machine_name;
+      $callback = $class_name::$callback;
+      $callback_path = $class_name::$callback_path;
+      $callback_module = $class_name::$callback_module;
+      $page_args = [];
       if ($class_name::$menu_path) {
         $menu_path = $class_name::$menu_path;
       }
+      if (!$callback) {
+        $callback = 'drupal_get_form';
+        $page_args = ['tripal_get_importer_form', $class_name];
+      }      
+      if (!$callback_path) {
+        $callback_path = 'includes/tripal.importer.inc';
+      }
+      $file_path = drupal_get_path('module', 'tripal');
+      if ($callback_path and $callback_module) {
+        $file_path = drupal_get_path('module', $callback_module);
+      }      
+      
       $items[$menu_path] = array(
         'title' => $class_name::$name,
         'description' =>  $class_name::$description,
-        'page callback' => 'drupal_get_form',
-        'page arguments' => array('tripal_get_importer_form', $class_name),
-        'access arguments' => array('load tripal data'),
+        'page callback' => $callback,
+        'page arguments' => $page_args,
+        'access arguments' => array('use ' . $machine_name . ' importer'),
         'type' => MENU_NORMAL_ITEM,
-        'file' => 'includes/tripal.importer.inc',
-        'file path' => drupal_get_path('module', 'tripal'),
+        'file' => $callback_path,
+        'file path' => $file_path,
       );
     }
   }
@@ -705,10 +695,6 @@ function tripal_permission() {
       'description' => t('Allows the user to publish Tripal content for online access.'),
       'restrict access' => TRUE,
     ),
-    'load tripal data' => array(
-      'title' => t('Load data into the Tripal site'),
-      'description' => t('Allows the user to load data into the Tripal site using data loaders. Some data loaders may have their own specific permission as well.'),
-    ),
     'upload files' => array(
       'title' => t('Upload Files'),
       'description' => t('Allows the user to upload files using Tripal\'s HTML5 loader.'),
@@ -746,6 +732,21 @@ function tripal_permission() {
       'restrict access' => TRUE,
     );
   }
+  
+  // Add permissions for each Importer
+  $importers = tripal_get_importers();
+  foreach ($importers as $class_name) {
+    tripal_load_include_importer_class($class_name);
+    if (class_exists($class_name)) {
+      $machine_name = $class_name::$machine_name;
+      $name = $class_name::$name;
+      $permissions['use ' . $machine_name . ' importer'] = array(
+        'title' => t('Importer: Use the %label', array('%label' => $name)),
+        'description' => t('Allow the user to import data using the  %label.  Note: you may also need to give the "Upload Files" permission for importers to work.', array('%label' => $name)),
+        'restrict access' => TRUE,
+      );
+    }
+  }
   return $permissions;
 }
 

tripal_chado/includes/TripalImporter/PubBulkImporter.inc

@@ -0,0 +1,132 @@
+<?php
+
+class PubBulkImporter extends TripalImporter {
+  /**
+   * The name of this loader.  This name will be presented to the site
+   * user.
+   */
+  public static $name = 'Chado Bulk Publication Importer';
+
+  /**
+   * The machine name for this loader. This name will be used to construct
+   * the URL for the loader.
+   */
+  public static $machine_name = 'chado_pub_bulk';
+
+  /**
+   * A brief description for this loader.  This description will be
+   * presented to the site user.
+   */
+  public static $description = 'Create and modify importers that can connect to and retrieve publications from remote databases.';
+
+  /**
+   * An array containing the extensions of allowed file types.
+   */
+  public static $file_types = array();
+
+
+  /**
+   * Provides information to the user about the file upload.  Typically this
+   * may include a description of the file types allowed.
+   */
+  public static $upload_description = '';
+
+  /**
+   * The title that should appear above the upload button.
+   */
+  public static $upload_title = 'File Upload';
+
+  /**
+   * If the loader should require an analysis record.  To maintain provenance
+   * we should always indiate where the data we are uploading comes from.
+   * The method that Tripal attempts to use for this by associating upload files
+   * with an analysis record.  The analysis record provides the details for
+   * how the file was created or obtained. Set this to FALSE if the loader
+   * should not require an analysis when loading. if $use_analysis is set to
+   * true then the form values will have an 'analysis_id' key in the $form_state
+   * array on submitted forms.
+   */
+  public static $use_analysis = FALSE;
+
+  /**
+   * If the $use_analysis value is set above then this value indicates if the
+   * analysis should be required.
+   */
+  public static $require_analysis = FALSE;
+
+  /**
+   * Text that should appear on the button at the bottom of the importer
+   * form.
+   */
+  public static $button_text = 'Import';
+
+  /**
+   * Indicates the methods that the file uploader will support.
+   */
+  public static $methods = array(
+    // Allow the user to upload a file to the server.
+    'file_upload' => FALSE,
+    // Allow the user to provide the path on the Tripal server for the file.
+    'file_local' => FALSE,
+    // Allow the user to provide a remote URL for the file.
+    'file_remote' => FALSE,
+  );
+
+  /**
+   * Indicates if the file must be provided.  An example when it may not be
+   * necessary to require that the user provide a file for uploading if the
+   * loader keeps track of previous files and makes those available for
+   * selection.
+   */
+  public static $file_required = FALSE;
+
+
+  /**
+   * The array of arguments used for this loader.  Each argument should
+   * be a separate array containing a machine_name, name, and description
+   * keys.  This information is used to build the help text for the loader.
+   */
+  public static $argument_list = array();
+
+
+  /**
+   * Indicates how many files are allowed to be uploaded.  By default this is
+   * set to allow only one file.  Change to any positive number. A value of
+   * zero indicates an unlimited number of uploaded files are allowed.
+   */
+  public static $cardinality = 0;
+  
+  /**
+   * Be default, all loaders are automatically added to the Admin >
+   * Tripal > Data Loaders menu.  However, if this loader should be
+   * made available via a different menu path, then set it here.  If the
+   * value is empty then the path will be the default.
+   */
+  public static $menu_path = '';
+
+  /**
+   * If your importer requires more flexibility and advance features than
+   * the TripalImporter provides you can indicate a callback function. If set,
+   * the callback will be used to provide the importer interface to the
+   * end-user.  However, because this bypasses the class infrastructure the
+   * run() function will also not be available and your importer must be
+   * fully self-sufficient outside of this class.  The benefit for using a
+   * TripalImporter despite your loader being self-sufficient is that Tripal
+   * will treat your loader like all others providing a consistent location
+   * in the menu and set of permissions.
+   */
+  public static $callback = 'tripal_pub_importers_list';
+  
+  /**
+   * The name of the module that provides the callback function.
+   */
+  public static $callback_module = 'tripal_chado';
+  
+  /**
+   * An include path for the callback function.  Use a relative path within
+   * this scope of this module 
+   * (e.g. includes/loaders/tripal_chado_pub_importers).
+   */
+  public static $callback_path = 'includes/loaders/tripal_chado.pub_importers.inc';
+
+}

tripal_chado/includes/setup/tripal_chado.setup.inc

@@ -225,8 +225,10 @@ function tripal_chado_prepare_chado($job = NULL) {
     }
 
     drush_print("Creating common Tripal Content Types...");
+    drush_print("This may take awhile if you are upgrading a site that has lots of data...");
 
     // Create the 'Organism' entity type. This uses the obi:organism term.
+    drush_print("Creating Organism...");
     $error = '';
     $args = array(
       'vocabulary' => 'OBI',
@@ -251,6 +253,7 @@ function tripal_chado_prepare_chado($job = NULL) {
     }
 
     // Create the 'Analysis' entity type. This uses the local:analysis term.
+    drush_print("Creating Analysis...");
     $error = '';
     $args = array(
       'vocabulary' => 'operation',
@@ -275,6 +278,7 @@ function tripal_chado_prepare_chado($job = NULL) {
     }
 
     // Create the 'Project' entity type. This uses the local:project term.
+    drush_print("Creating Project...");
     $error = '';
     $args = array(
       'vocabulary' => 'local',
@@ -299,6 +303,7 @@ function tripal_chado_prepare_chado($job = NULL) {
     }
 
     // Create the 'Map' entity type. This uses the local:project term.
+    drush_print("Creating Map...");
     $error = '';
     $args = array(
       'vocabulary' => 'data',
@@ -334,6 +339,7 @@ function tripal_chado_prepare_chado($job = NULL) {
     chado_import_pub_by_dbxref('PMID:24163125');
 
     // Create the 'Publication' entity type.
+    drush_print("Creating Publication...");
     $error = '';
     $args = array(
       'vocabulary' => 'TPUB',
@@ -377,6 +383,7 @@ function tripal_chado_prepare_chado($job = NULL) {
     }
 
     // Create the 'Gene' entity type.
+    drush_print("Creating Gene...");
     $error = '';
     $args = array(
       'vocabulary' => 'SO',
@@ -402,6 +409,7 @@ function tripal_chado_prepare_chado($job = NULL) {
     }
 
     // Create the 'mRNA' entity type.
+    drush_print("Creating mRNA...");
     $error = '';
     $args = array(
       'vocabulary' => 'SO',
@@ -427,6 +435,7 @@ function tripal_chado_prepare_chado($job = NULL) {
     }
 
     // Create the 'biological sample' entity type.
+    drush_print("Creating Biological Sample...");
     $error = '';
     $args = array(
       'vocabulary' => 'sep',
@@ -451,6 +460,7 @@ function tripal_chado_prepare_chado($job = NULL) {
     }
 
     // Create the 'Phylogenetic tree' entity type.
+    drush_print("Creating Phylogenetic tree...");
     $error = '';
     $args = array(
       'vocabulary' => 'data',
@@ -549,7 +559,7 @@ function tripal_chado_fix_v1_3_custom_tables() {
 
   // Update the featuremap_dbxref table by adding an is_current field.
   if (!chado_column_exists('featuremap_dbxref', 'is_current')) {
-    chado_query("ALTER TABLE {featuremap_dbxref} ADD COLUMN is_current boolean DEFAULT true NOT NULL,;");
+    chado_query("ALTER TABLE {featuremap_dbxref} ADD COLUMN is_current boolean DEFAULT true NOT NULL;");
   }
 
   // Remove the previously managed custom tables from the

tripal_chado/includes/tripal_chado.fields.inc

@@ -1458,6 +1458,9 @@ function tripal_chado_bundle_instances_info_custom(&$info, $entity_type, $bundle
         'chado_table' => $table_name,
         'chado_column' => $table_column,
         'base_table' => $table_name,
+        'term_accession' => '0100026',
+        'term_vocabulary' => 'OBI',
+        'term_name' => 'organism',
       ),
       'widget' => array(
         'type' => 'obi__organism_widget',
@@ -1490,6 +1493,9 @@ function tripal_chado_bundle_instances_info_custom(&$info, $entity_type, $bundle
         'chado_table' => 'biomaterial',
         'chado_column' => 'biosourceprovider_id',
         'base_table' => 'biomaterial',
+        'term_accession' => 'contact',
+        'term_vocabulary' => 'local',
+        'term_name' => 'contact',
       ),
       'widget' => array(
         'type' => 'local__contact_widget',
@@ -1557,6 +1563,9 @@ function tripal_chado_bundle_instances_info_custom(&$info, $entity_type, $bundle
         'chado_table' => $table_name,
         'chado_column' => 'cv_id',
         'base_table' => $table_name,
+        'term_accession' => '001080',
+        'term_vocabulary' => 'SIO',
+        'term_name' => 'vocabulary',
       ),
       'widget' => array(
         'type' => 'sio__vocabulary_widget',
@@ -1595,6 +1604,9 @@ function tripal_chado_bundle_instances_info_custom(&$info, $entity_type, $bundle
         'chado_table' => $table_name,
         'chado_column' => 'dbxref_id',
         'base_table' => $table_name,
+        'term_accession' => '2091',
+        'term_vocabulary' => 'data',
+        'term_name' => 'Accession',
       ),
       'widget' => array(
         'type' => 'data__accession_widget',
@@ -1631,6 +1643,9 @@ function tripal_chado_bundle_instances_info_custom(&$info, $entity_type, $bundle
         'chado_table' => $table_name,
         'chado_column' => 'md5checksum',
         'base_table' => $table_name,
+        'term_accession' => '2190',
+        'term_vocabulary' => 'data',
+        'term_name' => 'Sequence checksum',
       ),
       'widget' => array(
         'type' => 'data__sequence_checksum_widget',
@@ -1664,6 +1679,9 @@ function tripal_chado_bundle_instances_info_custom(&$info, $entity_type, $bundle
         'chado_table' => $table_name,
         'chado_column' => 'residues',
         'base_table' => $table_name,
+        'term_accession' => '2044',
+        'term_vocabulary' => 'data',
+        'term_name' => 'Sequence',
       ),
       'widget' => array(
         'type' => 'data__sequence_widget',
@@ -1699,6 +1717,9 @@ function tripal_chado_bundle_instances_info_custom(&$info, $entity_type, $bundle
         'chado_table' => $table_name,
         'chado_column' => 'seqlen',
         'base_table' => $table_name,
+        'term_accession' => '1249',
+        'term_vocabulary' => 'data',
+        'term_name' => 'delete	Sequence length',
       ),
       'widget' => array(
         'type' => 'data__sequence_length_widget',
@@ -1718,7 +1739,7 @@ function tripal_chado_bundle_instances_info_custom(&$info, $entity_type, $bundle
 
   // PROTEIN & CDS SEQUENCES.
   if ($table_name == 'feature' and
-     ($bundle->label == 'mRNA' or $bundle->label == 'transcript' or $bundle->label == 'Transcript'))  {
+      ($bundle->label == 'mRNA' or $bundle->label == 'transcript'))  {
     $field_name = 'data__protein_sequence';
     $info[$field_name] = array(
       'field_name' => $field_name,
@@ -1732,6 +1753,9 @@ function tripal_chado_bundle_instances_info_custom(&$info, $entity_type, $bundle
         'chado_table' => 'feature',
         'chado_column' => 'residues',
         'base_table' => 'feature',
+        'term_accession' => '2976',
+        'term_vocabulary' => 'data',
+        'term_name' => 'Protein sequence',
       ),
       'widget' => array(
         'type' => 'data__protein_sequence_widget',
@@ -1868,6 +1892,9 @@ function tripal_chado_bundle_instances_info_custom(&$info, $entity_type, $bundle
         'chado_table' => $table_name,
         'chado_column' => 'unittype_id',
         'base_table' => $table_name,
+        'term_accession' => '0000000',
+        'term_vocabulary' => 'UO',
+        'term_name' => 'unit',
       ),
       'widget' => array(
         'type' => 'uo__unit_widget',
@@ -1900,6 +1927,9 @@ function tripal_chado_bundle_instances_info_custom(&$info, $entity_type, $bundle
         'chado_table' => $table_name,
         'chado_column' => 'analysis_id',
         'base_table' => $table_name,
+        'term_accession' => 'source_data',
+        'term_vocabulary' => 'local',
+        'term_name' => 'source_data',
       ),
       'widget' => array(
         'type' => 'local__source_data_widget',
@@ -1958,6 +1988,9 @@ function tripal_chado_bundle_instances_info_custom(&$info, $entity_type, $bundle
         'chado_table' => 'phylotree',
         'chado_column' => 'type_id',
         'base_table' => 'phylotree',
+        'term_accession' => '0567',
+        'term_vocabulary' => 'operation',
+        'term_name' => 'Phylogenetic tree visualisation',
       ),
       'widget' => array(
         'type' => 'operation__phylotree_vis_widget',
@@ -1999,6 +2032,9 @@ function tripal_chado_bundle_instances_info_custom(&$info, $entity_type, $bundle
         'chado_table' => $table_name,
         'chado_column' => $table_column,
         'base_table' => $table_name,
+        'term_accession' => '00101',
+        'term_vocabulary' => 'sep',
+        'term_name' => 'Protocol',
       ),
       'widget' => array(
         'type' => 'sep__protocol_widget',
@@ -2077,7 +2113,6 @@ function tripal_chado_bundle_instances_info_custom(&$info, $entity_type, $bundle
         'term_vocabulary' => 'NCIT',
         'term_name' => 'Operator',
         'term_accession' => 'C48036',
-        
       ),
       'widget' => array(
         'type' => 'local__contact_widget',
@@ -2199,6 +2234,9 @@ function tripal_chado_bundle_instances_info_linker(&$info, $entity_type, $bundle
         'chado_table' => $contact_table,
         'base_table' => $table_name,
         'chado_column' => 'contact_id',
+        'term_accession' => 'contact',
+        'term_vocabulary' => 'local',
+        'term_name' => 'contact',
       ),
       'widget' => array(
         'type' => 'local__contact_widget',
@@ -2274,6 +2312,9 @@ function tripal_chado_bundle_instances_info_linker(&$info, $entity_type, $bundle
 //         'chado_table' => $expression_table,
 //         'chado_column' => $pkey,
 //         'base_table' => $table_name,
+//         'term_accession' => '',
+//         'term_vocabulary' => '',
+//         'term_name' => '',
 //       ),
 //       'widget' => array(
 //         'type' => 'go__gene_expression_widget',
@@ -2309,6 +2350,9 @@ function tripal_chado_bundle_instances_info_linker(&$info, $entity_type, $bundle
         'chado_table' => 'featureloc',
         'chado_column' => $pkey,
         'base_table' => 'feature',
+        'term_accession' => '2012',
+        'term_vocabulary' => 'data',
+        'term_name' => 'Sequence coordinates',
       ),
       'widget' => array(
         'type' => 'data__sequence_coordinates_widget',
@@ -2343,6 +2387,9 @@ function tripal_chado_bundle_instances_info_linker(&$info, $entity_type, $bundle
         'chado_table' => 'featurepos',
         'chado_column' => $pkey,
         'base_table' => 'feature',
+        'term_accession' => '0000021',
+        'term_vocabulary' => 'OGI',
+        'term_name' => 'location on map',
       ),
       'widget' => array(
         'type' => 'ogi__location_on_map_widget',
@@ -2378,6 +2425,9 @@ function tripal_chado_bundle_instances_info_linker(&$info, $entity_type, $bundle
 //         'chado_table' => $genotype_table,
 //         'chado_column' => $pkey,
 //         'base_table' => $table_name,
+//         'term_accession' => '',
+//         'term_vocabulary' => '',
+//         'term_name' => '',
 //       ),
 //       'widget' => array(
 //         'type' => 'so__genotype_widget',
@@ -2413,6 +2463,9 @@ function tripal_chado_bundle_instances_info_linker(&$info, $entity_type, $bundle
 //         'chado_table' => $phenotype_table,
 //         'chado_column' => $pkey,
 //         'base_table' => $table_name,
+//         'term_accession' => '',
+//         'term_vocabulary' => '',
+//         'term_name' => '',
 //       ),
 //       'widget' => array(
 //         'type' => 'sbo__phenotype_widget',
@@ -2673,6 +2726,9 @@ function tripal_chado_bundle_instances_info_linker(&$info, $entity_type, $bundle
         'chado_table' => $table_name,
         'chado_column' => $pkey,
         'base_table' => $table_name,
+        'term_accession' => '000631',
+        'term_vocabulary' => 'SIO',
+        'term_name' => 'references',
       ),
       'widget' => array(
         'type' => 'sio__references_widget',
@@ -2747,6 +2803,9 @@ function tripal_chado_bundle_instances_info_linker(&$info, $entity_type, $bundle
         'chado_table' => $syn_table,
         'chado_column' => $pkey,
         'base_table' => $table_name,
+        'term_accession' => 'alternateName',
+        'term_vocabulary' => 'schema',
+        'term_name' => 'alternateName',
       ),
       'widget' => array(
         'type' => 'schema__alternate_name_widget',

tripal_chado/includes/tripal_chado.vocab_storage.inc

@@ -230,9 +230,9 @@ function tripal_chado_vocab_get_term_children($vocabulary, $accession) {
   $cvterm = chado_expand_var($cvterm, 'field', 'cvterm.definition');
 
 
-  // Get the children
+  // Get the children.
   $sql = "
-    SELECT subject_id
+    SELECT DISTINCT subject_id
     FROM {cvterm_relationship} CVTR
     WHERE object_id = :object_id
   ";

tripal_chado/tripal_chado.module

@@ -386,13 +386,13 @@ function tripal_chado_menu() {
   //////////////////////////////////////////////////////////////////////////////
 
   $items['admin/tripal/loaders/pub'] = array(
-    'title' => t('Chado Publication Importers'),
+    'title' => t('Chado Bulk Publication Importer Callback'),
     'description' => t('Create and modify importers that can connect to and retreive publications from remote databases.'),
     'page callback' => 'tripal_pub_importers_list',
-    'access arguments' => array('load tripal data'),
+    'access arguments' => array('use chado_pub_bulk importer'),
     'file' => 'includes/loaders/tripal_chado.pub_importers.inc',
     'file path' => drupal_get_path('module', 'tripal_chado'),
-    'type' => MENU_NORMAL_ITEM,
+    'type' => MENU_CALLBACK,
     'weight' => 0
   );
 
@@ -400,7 +400,7 @@ function tripal_chado_menu() {
     'title' => t('Add an Importer'),
     'description' => t('Add a new publication importer.'),
     'page callback' => 'tripal_pub_importer_setup_page',
-    'access arguments' => array('load tripal data'),
+    'access arguments' => array('use chado_pub_bulk importer'),
     'type ' => MENU_CALLBACK,
     'file' => 'includes/loaders/tripal_chado.pub_importers.inc',
     'file path' => drupal_get_path('module', 'tripal_chado'),
@@ -408,7 +408,7 @@ function tripal_chado_menu() {
   $items['admin/tripal/loaders/pub/edit/%'] = array(
     'page callback' => 'tripal_pub_importer_setup_page',
     'page arguments' => array('edit', 5),
-    'access arguments' => array('load tripal data'),
+    'access arguments' => array('use chado_pub_bulk importer'),
     'type ' => MENU_CALLBACK,
     'file' => 'includes/loaders/tripal_chado.pub_importers.inc',
     'file path' => drupal_get_path('module', 'tripal_chado'),
@@ -416,7 +416,7 @@ function tripal_chado_menu() {
   $items['admin/tripal/loaders/pub/raw/%'] = array(
     'page callback' => 'tripal_get_remote_pub_raw_page',
     'page arguments' => array(5),
-    'access arguments' => array('load tripal data'),
+    'access arguments' => array('use chado_pub_bulk importer'),
     'type ' => MENU_CALLBACK,
     'file' => 'includes/loaders/tripal_chado.pub_importers.inc',
     'file path' => drupal_get_path('module', 'tripal_chado'),
@@ -425,7 +425,7 @@ function tripal_chado_menu() {
   // add a second link for the importer on the data loaders page
   $items['admin/tripal/loaders/pub/import'] = array(
     'page callback' => 'tripal_pub_importers_list',
-    'access arguments' => array('load tripal data'),
+    'access arguments' => array('use chado_pub_bulk importer'),
     'type' => MENU_CALLBACK,
     'file' => 'includes/loaders/tripal_chado.pub_importers.inc',
     'file path' => drupal_get_path('module', 'tripal_chado'),
@@ -434,7 +434,7 @@ function tripal_chado_menu() {
   $items['admin/tripal/loaders/pub/submit/%'] = array(
     'page callback' => 'tripal_pub_importer_submit_job',
     'page arguments' => array(5),
-    'access arguments' => array('load tripal data'),
+    'access arguments' => array('use chado_pub_bulk importer'),
     'type ' => MENU_CALLBACK,
     'file' => 'includes/loaders/tripal_chado.pub_importers.inc',
     'file path' => drupal_get_path('module', 'tripal_chado'),
@@ -443,7 +443,7 @@ function tripal_chado_menu() {
   $items['admin/tripal/loaders/pub/delete/%'] = array(
     'page callback' => 'tripal_pub_importer_delete',
     'page arguments' => array(5),
-    'access arguments' => array('load tripal data'),
+    'access arguments' => array('use chado_pub_bulk importer'),
     'type ' => MENU_CALLBACK,
     'file' => 'includes/loaders/tripal_chado.pub_importers.inc',
     'file path' => drupal_get_path('module', 'tripal_chado'),
@@ -451,7 +451,7 @@ function tripal_chado_menu() {
   $items['admin/tripal/loaders/pub/changedb'] = array(
     'page callback' => 'tripal_pub_importer_setup_page_update_remotedb',
     'page arguments' => array(),
-    'access arguments' => array('load tripal data'),
+    'access arguments' => array('use chado_pub_bulk importer'),
     'type ' => MENU_CALLBACK,
     'file' => 'includes/loaders/tripal_chado.pub_importers.inc',
     'file path' => drupal_get_path('module', 'tripal_chado'),
@@ -460,7 +460,7 @@ function tripal_chado_menu() {
   $items['admin/tripal/loaders/pub/criteria/%/%'] = array(
     'page callback' => 'tripal_pub_importer_setup_page_update_criteria',
     'page arguments' => array(5, 6),
-    'access arguments' => array('load tripal data'),
+    'access arguments' => array('use chado_pub_bulk importer'),
     'type ' => MENU_CALLBACK,
     'file' => 'includes/loaders/tripal_chado.pub_importers.inc',
     'file path' => drupal_get_path('module', 'tripal_chado'),
@@ -856,7 +856,6 @@ function tripal_chado_permission() {
       'title' => t('Administer Semantic Web and Chado Integration'),
       'description' => t('Allows the user to assign controlled vocabulary terms to tables and table columns in Chado.')
     ),
-
   );
 }
 

tripal_ws/api/tripal_ws.api.inc

@@ -194,6 +194,36 @@ function tripal_remove_site($record_id) {
   return FALSE;
 }
 
+/**
+ * Constructs a URL for a remote Tripal web service.
+ * 
+ * @param $remote_site
+ *   A remote Tripal site object.
+ * @param $path
+ *   The web service path for the content (excluding
+ *   'web-servcies/vX.x/content').  To retrieve the full content listing
+ *   leave this paramter empty.
+ * @param $query
+ *   An query string to appear after the ? in a URL.
+ *   
+ * @return  
+ *   The full URL within the content service.
+ */
+function tripal_build_remote_content_url($remote_site, $path = '', $query = '') {
+  // Build the URL to the remote web services.
+  $ws_version = $remote_site->version;
+  $ws_url = $remote_site->url;
+  $ws_url = trim($ws_url, '/');
+  $ws_url .= '/web-services/content/' . $ws_version . '/' . $path;
+  
+  // Build the Query and make and substitions needed.
+  if ($query) {
+    $ws_url = $ws_url . '?' . $query;
+  }
+  
+  return $ws_url;
+}
+
 /**
  * Makes a request to the "content" service of a remote Tripal web site.
  *
@@ -225,57 +255,61 @@ function tripal_get_remote_content($site_id, $path = '', $query = '') {
     ->fetchObject();
 
   if (!$remote_site) {
-    tripal_report_error('tripal_ws', TRIPAL_ERROR,
-      t('Could not find a remote tripal site using the id provided: !id.',
-        array('!id' => $site_id)));
-    return FALSE;
-  }
-
-  // Build the URL to the remote web services.
-  $ws_version = $remote_site->version;
-  $ws_url = $remote_site->url;
-  $ws_url = trim($ws_url, '/');
-  $ws_url .= '/web-services/content/' . $ws_version . '/' . $path;
-
-  // Build the Query and make and substitions needed.
-  if ($query) {
-    $ws_url = $ws_url . '?' . $query;
-  }
-
-  // TODO: something is wrong here, the query is not being recognized on
-  // the remote Tripal site. It's just returning the default.
+    $data = [
+      'error' => t('Could not find a remote tripal site using the id provided: !id.',
+        array('!id' => $site_id))
+    ];
+    _tripal_report_ws_error($data);
+    return $data;
+  }
+  
+  // Make the remote query.
+  $ws_url = tripal_build_remote_content_url($remote_site, $path, $query);
   $data = drupal_http_request($ws_url);
-
   if (!$data) {
-    tripal_report_error('tripal_ws', TRIPAL_ERROR,
-        t('Could not connect to the remote web service.'));
-    return FALSE;
+    $data = [
+      'error' => t('Could not connect to the remote web service using the url: !url',
+        ['!url' => $ws_url])
+    ];
+    _tripal_report_ws_error($data);
+    return $data;
   }
 
   // If the data object has an error then this is some sort of
   // connection error (not a Tripal web servcies error).
   if (property_exists($data, 'error')) {
-    tripal_report_error('tripal_ws', TRIPAL_ERROR,
-        'Remote web Services reports the following error: !error. Using URL: !url',
-        array('!error' => $error, '!url' => $ws_url));
-    return FALSE;
+    $data = [
+      'error' => $data->error
+    ];
+    _tripal_report_ws_error($data);
+    return $data;
   }
 
   // We got a response, so convert it to a PHP array.
   $data = drupal_json_decode($data->data);
-
+  
   // Check if there was a Tripal Web Services error.
   if (array_key_exists('error', $data)) {
-    $error = '</pre>' . print_r($data['error'], TRUE) . '</pre>';
-    tripal_report_error('tripal_ws', TRIPAL_ERROR,
-        'Tripal remote web services reports the following error: !error. Using URL: !url',
-        array('!error' => $error, '!url' => $ws_url));
-    return FALSE;
+    _tripal_report_ws_error($data);
   }
 
   return $data;
 }
 
+/**
+ * A helper function for reporting an error when retrieving remote content.
+ * 
+ * @param $data
+ *   A data array containing at a minimum the 'error' key containing the
+ *   error message.
+ */
+function _tripal_report_ws_error($data) {  
+  $error = '</pre>' . print_r($data['error'], TRUE) . '</pre>';
+  tripal_report_error('tripal_ws', TRIPAL_ERROR,
+    'Tripal remote web services reports the following error: !error.',
+    array('!error' => $error));
+}
+
 /**
  * Retrieves the JSON-LD context for any remote Tripal web service.
  *
@@ -497,7 +531,9 @@ function tripal_load_remote_entities($remote_entity_ids, $site_id, $bundle_acces
     '&fields=' . urlencode(implode(",", $field_ids));
 
   $results = tripal_get_remote_content($site_id, $bundle_accession, $query);
-  if (!$results) {
+  
+  // If we encountered an error just return;
+  if (array_key_exists('error', $results)) {
     return FALSE;
   }
 
@@ -565,7 +601,9 @@ function tripal_load_remote_entity($remote_entity_id, $site_id, $bundle_accessio
 
   // Get the remote entity and create the fake entity.
   $remote_entity = tripal_get_remote_content($site_id, $bundle_accession . '/' . $remote_entity_id);
-  if (!$remote_entity) {
+  
+  // If we encountered an error just return;
+  if (array_key_exists('error', $results)) {
     return FALSE;
   }
 

tripal_ws/includes/TripalFields/remote__data/remote__data.inc

@@ -89,7 +89,7 @@ class remote__data extends WebServicesField {
   // should exclude the field from web services or downloads.  An example
   // could be a quick search field that appears on the page that redirects
   // the user but otherwise provides no data.
-  public static $no_data = FALSE;
+  public static $no_data = TRUE;
 
   // Holds an object describing the remote site that tihs field connects to.
   private $remote_site = NULL;
@@ -105,9 +105,8 @@ class remote__data extends WebServicesField {
     // We don't want remote content to be available in web services.  There
     // is an if statement to not show this field in the web services but the
     // entity_load function doesn't know this field shouldn't be loaded so
-    // we need to short-circuit that.
-    $_SERVER['REQUEST_URI'];
-    if (preg_match('/^web-services/', $_SERVER['REQUEST_URI'])) {
+    // we need to short-circuit that.  
+    if (preg_match('/web-services/', $_SERVER['REQUEST_URI'])) {
       $this->loaded_via_ws = TRUE;
       return;
     }
@@ -128,22 +127,26 @@ class remote__data extends WebServicesField {
    * @see WebServicesField::load()
    */
   public function load($entity) {
+    
+    // If this field is being loaded via web services then just return.
+    if ($this->loaded_via_ws == TRUE) {
+      return;
+    }
 
     $field_name = $this->field['field_name'];
     $field_type = $this->field['type'];
 
     // Set some defaults for the empty record.
     $entity->{$field_name}['und'][0] = array(
-      'value' => array(),
-      'remote_entity' => array(),
+      'value' => '',
+      'remote_entity' => NULL,
+      'error' => FALSE,
+      'warning' => FALSE,
+      'admin_message' => '',
+      'query_str' => '',
     );
 
-    // If this field is being loaded via web services then just return.
-    if ($this->loaded_via_ws == TRUE) {
-      return;
-    }
-
-    // Get the query set by the admin for this field and replace any tokesn
+    // Get the query set by the admin for this field and replace any tokens
     $query_str = $this->instance['settings']['data_info']['query'];
     $bundle = tripal_load_bundle_entity(array('name' => $entity->bundle));
     $query_str = tripal_replace_entity_tokens($query_str, $entity, $bundle);
@@ -151,23 +154,47 @@ class remote__data extends WebServicesField {
     // Make the request.
     $data = $this->makeRemoteRequest($query_str);
     if(!$data){
+      $entity->{$field_name}['und'][0]['value'] = 'ERROR: there was a problem retrieving content for this field.';
+      $entity->{$field_name}['und'][0]['admin_message'] =  "The remote service returned no data.";
+      $entity->{$field_name}['und'][0]['remote_entity'] = NULL;
+      $entity->{$field_name}['und'][0]['error'] = TRUE;
+      $entity->{$field_name}['und'][0]['warning'] = FALSE;
+      $entity->{$field_name}['und'][0]['query_str'] = $this->buildRemoteURL($this->remote_site, $query_str);      
+      return;
+    }
+    // Make sure we didn't have a problem
+    if (array_key_exists('error', $data)) {
+      $entity->{$field_name}['und'][0]['value'] = 'ERROR: there was a problem retrieving content for this field.';
+      $entity->{$field_name}['und'][0]['admin_message'] = "The  content is currently not available because the " .
+          "remote service reported the following error: " . $data['error'] . ".";
+      $entity->{$field_name}['und'][0]['remote_entity'] = NULL;
+      $entity->{$field_name}['und'][0]['error'] = TRUE;
+      $entity->{$field_name}['und'][0]['warning'] = FALSE;
+      $entity->{$field_name}['und'][0]['query_str'] = $this->buildRemoteURL($this->remote_site, $query_str);
       return;
     }
 
-    $total_items = $data['totalItems'];
-
+    $num_items = count($data['member']);
+    if ($num_items == 0) {
+      $entity->{$field_name}['und'][0]['value'] = 'Content is unavailable on the remote service.';
+      $entity->{$field_name}['und'][0]['admin_message'] = "The query to the remote service returned an empty result set. If you " .
+          "think this is an error, please check the query string and the remote service to verify. ";
+      $entity->{$field_name}['und'][0]['warning'] = TRUE;
+      $entity->{$field_name}['und'][0]['error'] = FALSE;
+      $entity->{$field_name}['und'][0]['remote_entity'] = NULL;
+      $entity->{$field_name}['und'][0]['query_str'] = $this->buildRemoteURL($this->remote_site, $query_str);
+      return;
+    }
+    
     // Iterate through the members returned and save those for the field.
-    for ($i = 0; $i < count($data['members']); $i++) {
-      $member = $data['members'][$i];
+    for ($i = 0; $i < $num_items; $i++) {
+      $member = $data['member'][$i];
 
       // Get the cotent type and remote entity id
       $content_type = $member['@type'];
       $remote_entity_id = $member['@id'];
       $remote_entity_id = preg_replace('/^.*\/(\d+)/', '$1', $remote_entity_id);
 
-      // Save the member information for use later.
-      $entity->{$field_name}['und'][$i]['remote_entity'] = $member;
-
       // Separate the query_field if it has subfields.
       $rd_field_name = $this->instance['settings']['data_info']['rd_field_name'];
       $subfields = explode(',', $rd_field_name);
@@ -176,21 +203,42 @@ class remote__data extends WebServicesField {
       // Next get the the details about this member.
       $query_field_url =  $content_type . '/' . $remote_entity_id . '/' . $query_field;
       $field_data = $this->makeRemoteRequest($query_field_url);
-      if(!$field_data){
-        // If we encounter any type of error, we'll reset the field and return.
-        $entity->{$field_name}['und'] = array();
-        $entity->{$field_name}['und'][0] = array(
-          'value' => array(),
-          'remote_entity' => array(),
-        );
+      
+      // If we encounter any type of error, we'll reset the field and return.
+      if (array_key_exists('error', $field_data)) {
+        $entity->{$field_name} = [];
+        $entity->{$field_name}['und'][0]['value'] = 'ERROR: there was a problem retrieving secific content for this field.';
+        $entity->{$field_name}['und'][0]['admin_message'] = "While iterating through the list of results, the " .
+          "remote service reported the following error: " . $field_data['error'] . ". " ;
+        $entity->{$field_name}['und'][0]['remote_entity'] = NULL;
+        $entity->{$field_name}['und'][0]['error'] = TRUE;
+        $entity->{$field_name}['und'][0]['warning'] = FALSE;
+        $entity->{$field_name}['und'][0]['query_str'] = $this->buildRemoteURL($this->remote_site, $query_field_url);
         return;
       }
-
+      
       // Set the field data as the value.
       $field_data_type = $field_data['@type'];
       $entity->{$field_name}['und'][$i]['value'] = $field_data;
+      $entity->{$field_name}['und'][$i]['remote_entity'] = $member;
+      $entity->{$field_name}['und'][$i]['error'] = FALSE;
+      $entity->{$field_name}['und'][$i]['warning'] = FALSE;
+      $entity->{$field_name}['und'][$i]['admin_message'] = '';
+      $entity->{$field_name}['und'][$i]['query_str'] = $this->buildRemoteURL($this->remote_site, $query_field_url);;
     }
-   }
+  }
+  
+  /**
+   * Used to build the full URL for the query.
+   */
+  private function buildRemoteURL($remote_site, $query) {
+    $path = $query;
+    $q = '';
+    if (preg_match('/\?/', $query)) {
+      list($path, $q) = explode('?', $query);
+    }
+    return tripal_build_remote_content_url($remote_site, $path, $q);
+  }
    /**
     * Makes a request to a remote Tripal web services site.
     *
@@ -199,13 +247,12 @@ class remote__data extends WebServicesField {
     *   website.
     */
    private function makeRemoteRequest($query) {
-     $ctype = $query;
-     $qdata = '';
+     $path = $query;
+     $q = '';
      if (preg_match('/\?/', $query)) {
-       list($ctype, $qdata) = explode('?', $query);
+       list($path, $q) = explode('?', $query);
      }
-
-     $data = tripal_get_remote_content($this->remote_site->id, $query);
+     $data = tripal_get_remote_content($this->remote_site->id, $path, $q);
 
      return $data;
    }
@@ -259,8 +306,20 @@ class remote__data extends WebServicesField {
     $element['data_info']['query'] = array(
       '#type' => 'textarea',
       '#title' => 'Query to Execute',
-      '#description' => 'Build the query string that should be appended after the url. The tokens
-      listed below may be used in your query build.',
+      '#description' => 'Enter the query that will retreive the remote records. ' . 
+        'If the full URL to the content web service is ' .
+        'https://[tripal_site]/web-services/content/v0.1/. Then this field should ' .
+        'contain the text immediately after the content/v0.1 portion of the URL. ' .
+        'For information about building web services queries see the ' .
+        'online documentation at ' . l('The Tripal v3 User\'s Guide', 'http://tripal.info/tutorials/v3.x/web-services') . '. ' . 
+        'For example, suppose this field is attached to an ' .
+        'Organism content type on the local site, and you want to retrieve a ' .
+        'field for the same organism on a remote Tripal site then you will ' .
+        'want to query on the genus and species. Also, you want the genus and ' .
+        'species to match the organism that this field is attached to. You can ' .
+        'use tokens to do this (see the "Available Tokesn" fieldset below). ' .
+        'For this example, the query text should be ' . 
+        'Organism?genus=[taxrank__genus]&species=[taxrank__species].',
       '#default_value' => $this->instance['settings']['data_info']['query'],
       '#rows' => 5,
       '#required' => TRUE

tripal_ws/includes/TripalFields/remote__data/remote__data_formatter.inc

@@ -11,64 +11,11 @@ class remote__data_formatter extends WebServicesFieldFormatter {
     'setting1' => 'default_value',
   );
   /**
-   * Provides the field's setting form.
-   *
-   * This function corresponds to the hook_field_formatter_settings_form()
-   * function of the Drupal Field API.
-   *
-   * The settings form appears on the 'Manage Display' page of the content
-   * type administration page. This function provides the form that will
-   * appear on that page.
-   *
-   * To add a validate function, please create a static function in the
-   * implementing class, and indicate that this function should be used
-   * in the form array that is returned by this function.
-   *
-   * This form will not be displayed if the formatter_settings_summary()
-   * function does not return anything.
-   *
-   * param $field
-   *   The field structure being configured.
-   * param $instance
-   *   The instance structure being configured.
-   * param $view_mode
-   *   The view mode being configured.
-   * param $form
-   *   The (entire) configuration form array, which will usually have no use
-   *   here.  Typically for reference only.
-   * param $form_state
-   *   The form state of the (entire) configuration form.
-   *
-   * @return
-   *   A Drupal Form array containing the settings form for this field.
-   */
-  public function settingsForm($view_mode, $form, &$form_state) {
-  }
-  /**
-   *  Provides the display for a field
-   *
-   * This function corresponds to the hook_field_formatter_view()
-   * function of the Drupal Field API.
-   *
-   *  This function provides the display for a field when it is viewed on
-   *  the web page.  The content returned by the formatter should only include
-   *  what is present in the $items[$delta]['values] array. This way, the
-   *  contents that are displayed on the page, via webservices and downloaded
-   *  into a CSV file will always be identical.  The view need not show all
-   *  of the data in the 'values' array.
-   *
-   *  @param $element
-   *  @param $entity_type
-   *  @param $entity
-   *  @param $langcode
-   *  @param $items
-   *  @param $display
-   *
-   *  @return
-   *    An element array compatible with that returned by the
-   *    hook_field_formatter_view() function.
+   * @see TripalFieldFormatter::view()
    */
   public function view(&$element, $entity_type, $entity, $langcode, $items, $display) {
+    $content = '';
+    
     // Get the settings
     $settings = $display['settings'];
     $field_name = $this->field['field_name'];
@@ -94,25 +41,39 @@ class remote__data_formatter extends WebServicesFieldFormatter {
       ->execute()
       ->fetchObject();
 
-      $content = '<p>';
-    if (is_object($site_logo)) {
-      $content .= '<img class="tripal-remote--data-field-logo" src="' . file_create_url($site_logo->uri) . '"><br/>';
-    }
-    $content .=  t('This content provided by !site.',
-        array('!site' => l($site->name, $site->url, array('attributes' => array("target" => '_blank')))));
-    $content .= '</p>';
+    // Iterate through the results and create a generic table.
     $rows = array();
+    $headers = array('');
     foreach ($items as $index => $item) {
-      $remote_entity_label = $item['remote_entity']['label'];
-      $remote_entity_page = $item['remote_entity']['ItemPage'];
-      $link = t('View !data on %site',
-          array('!data' => l('this content', $remote_entity_page, array('attributes' => array('target' => '_blank'))),
-            '%site' => $site->name));
+      if (!$item['value'] or empty($item['value'])) {
+        continue;
+      }
       $value = $item['value'];
-      if (!$value) {
+      $error = $item['error'];
+      $warning = $item['warning'];
+      
+      // If there is an error or warning then clear the cache for this field
+      // so that next time the page is loaded it will try to reload again.
+      if ($error or $warning) {
+        $cid = "field:TripalEntity:" . $entity->id . ':' . $field_name;
+        cache_clear_all($cid, 'cache_field');
+        if ($item['admin_message']) {
+          $severity = TRIPAL_ERROR;
+          if ($warning) {
+            $severity = TRIPAL_WARNING;
+          }
+          $value .= tripal_set_message($item['admin_message'] . 'The query URL was: ' . l($item['query_str'], $item['query_str'], ['attributes' => ['target' => '_blank']]),
+            $severity, ['return_html' => TRUE]);
+        }
+        $rows[] = [$value];
         continue;
       }
-      $headers = array('');
+      
+      $remote_entity_label = array_key_exists('label', $item) ? $item['remote_entity']['label'] : '';
+      $remote_entity_page = $item['remote_entity']['ItemPage'];
+      $remote_entity_link = t('View !data on %site',
+          array('!data' => l('this data', $remote_entity_page, array('attributes' => array('target' => '_blank'))),
+            '%site' => $site->name));
 
       // If this is a collection then handle it as a list of members.
       if (array_key_exists('members', $value)) {
@@ -128,7 +89,7 @@ class remote__data_formatter extends WebServicesFieldFormatter {
         }
         else {
           if (array_key_exists($flabel, $value)) {
-            $rows[] = array(l($value[$flabel], $remote_entity_page, array('attributes' => array('target' => '_blank'))));
+            $rows[] = array($value[$flabel]);
           }
           else {
             $value['Link'] = l('View content on ' . $site->name, $remote_entity_page, array('attributes' => array('target' => '_blank')));
@@ -136,8 +97,9 @@ class remote__data_formatter extends WebServicesFieldFormatter {
           }
         }
       }
-
     }
+    
+    // TODO: we need to handle paged elements.
 
     $has_sub_tables = FALSE;
     for ($i = 0; $i < count($rows); $i++) {
@@ -145,6 +107,12 @@ class remote__data_formatter extends WebServicesFieldFormatter {
         $rows[$i][0] = $this->createTable($rows[$i]);
         $has_sub_tables = TRUE;
       }
+      else {
+        $rows[$i] = [
+          'colspan' => 2,
+          'data' => $rows[$i],
+        ];
+      }
     }
 
     // If we don't have  tables for each row then we'll put everything into
@@ -171,6 +139,19 @@ class remote__data_formatter extends WebServicesFieldFormatter {
         $content .= $rows[$i][0];
       }
     }
+    
+    $content .= '<p>';
+
+    $content .=  t('This content provided by !site.',
+      array('!site' => l($site->name, $site->url, array('attributes' => array("target" => '_blank')))));
+    if (is_object($site_logo)) {
+      $content .= '<img class="tripal-remote--data-field-logo" src="' . file_create_url($site_logo->uri) . '"><br/>';
+    }
+    if (count($items) == 1) {
+      $content .= $remote_entity_link;
+    }
+    $content .= '</p>';
+    
     // Return the content for this field.
     $element[0] = array(
       '#type' => 'markup',

tripal_ws/includes/TripalWebService/TripalContentService_v0_1.inc

@@ -68,10 +68,14 @@ class TripalContentService_v0_1 extends TripalWebService {
       $ctype_lookup = array();
       $found = FALSE;
       while ($bundle = $bundles->fetchObject()) {
-        if ($ctype == preg_replace('/[^\w]/', '_', $bundle->label)) {
+        // Check the label by replacing non alpha-numeric characters with 
+        // an underscore and is case-insensitive
+        $label = preg_replace('/[^\w]/', '_', $bundle->label);
+        if (preg_match("/^$label$/i", $ctype)) {
           $ctype = $bundle->label;
           $found = TRUE;
         }
+        // Check if this is an accession.
         if ($ctype == $bundle->vocabulary . ':' . $bundle->accession) {
           $ctype = $bundle->label;
           $found = TRUE;
@@ -274,8 +278,16 @@ class TripalContentService_v0_1 extends TripalWebService {
       }
       // Get the information about this field.
       $field = field_info_field($field_name);
+      
+      // If the field has the $no_data turned on then we should exclude it.
+      if (tripal_load_include_field_class($field['type'])) {
+        $field_class = $field['type'];
+        if ($field_class::$no_data) {
+          return;
+        }
+      }
 
-      // Skip the remote__data field that is provided by the tripal_Ws
+      // Skip the remote__data field that is provided by the tripal_ws
       // module.
       if ($field['type'] == 'remote__data') {
         continue;
@@ -946,12 +958,10 @@ class TripalContentService_v0_1 extends TripalWebService {
       $entity = $entity[$entity_id];
 
       // Add in any requested fields
-      foreach ($fields as $expfield) {
-        if (array_key_exists($expfield, $add_fields)) {
-          $this->addEntityField($member, $add_fields[$expfield]['term'], $entity,
-              $bundle, $add_fields[$expfield]['field'], $add_fields[$expfield]['instance'],
-              $service_path);
-        }
+      foreach ($add_fields as $expfield => $expfield_details) {
+        $this->addEntityField($member, $expfield_details['term'], $entity,
+            $bundle, $expfield_details['field'], $expfield_details['instance'],
+            $service_path);
       }
       $this->resource->addMember($member);
     }

tripal_ws/includes/tripal_ws.admin.inc

@@ -98,14 +98,14 @@ function tripal_ws_tripal_sites_edit_form($form, &$form_state, $tripal_site_id =
   $form['tripal_site_info']['url'] = array(
     '#title' => t('URL'),
     '#type' => 'textfield',
-    '#description' => t('The URL of the Tripal site.'),
+    '#description' => t('The URL of the Tripal site, including the "http://" or "https://" followed by the address for the site\'s home page.'),
     '#default_value' => $url,
     '#required' => TRUE
   );
   $form['tripal_site_info']['version'] = array(
     '#title' => t('Version'),
     '#type' => 'textfield',
-    '#description' => t('Web services version used by the Tripal site.'),
+    '#description' => t('Web services version used by the Tripal site (example: v0.1)'),
     '#default_value' => $version,
   );
   $form['tripal_site_info']['description'] = array(

tripal_ws/theme/css/tripal_ws.css

@@ -5,6 +5,7 @@
 }
 .tripal-remote--data-field-logo {
   max-height: 100px;
+  max-width: 150px;
 }
 
 .tripal-remote--data-field-dl dt {