Merge branch '7.x-3.x' into 620-tv3-performance_views_clear

legacy/tripal_contact/tripal_contact.install

@@ -51,14 +51,6 @@ function tripal_contact_requirements($phase) {
  */
 function tripal_contact_install() {
 
-  // Add the contactprop table to Chado.
-  tripal_contact_add_custom_tables();
-
-  // Add loading of the the tripal contact ontology to the job queue.
-  $obo_path = '{tripal_contact}/files/tcontact.obo';
-  $obo_id = tripal_insert_obo('Tripal Contacts', $obo_path);
-  tripal_submit_obo_job(array('obo_id' => $obo_id));
-
   // Add cvterms for relationship types.
   tripal_contact_add_cvs();
   tripal_contact_add_cvterms();
@@ -75,13 +67,7 @@ function tripal_contact_install() {
  * @ingroup tripal_legacy_contact
  */
 function tripal_contact_uninstall() {
-  /*
-  // remove our custom block visibility settings per node type
-  db_delete('block_node_type')
-    ->condition('module', 'chado_contact')
-    ->condition('delta', 'contbase')
-    ->execute();
-    */
+
 }
 
 /**
@@ -166,71 +152,3 @@ function tripal_contact_schema() {
   return $schema;
 }
 
-/**
- * Add any custom tables needed by this module.
- * - Contactprop: keep track of properties of contact
- *
- * @ingroup tripal_legacy_contact
- */
-// This function was moved to tripal_chado/includes/setup/tripal_chado.setup.inc
-/* function tripal_contact_add_custom_tables(){
-  $schema = array (
-    'table' => 'contactprop',
-    'fields' => array (
-      'contactprop_id' => array (
-        'type' => 'serial',
-        'not null' => true,
-      ),
-      'contact_id' => array (
-        'type' => 'int',
-        'not null' => true,
-      ),
-      'type_id' => array (
-        'type' => 'int',
-        'not null' => true,
-      ),
-      'value' => array (
-        'type' => 'text',
-        'not null' => false,
-      ),
-      'rank' => array (
-        'type' => 'int',
-        'not null' => true,
-        'default' => 0,
-      ),
-    ),
-    'primary key' => array (
-      0 => 'contactprop_id',
-    ),
-    'unique keys' => array (
-      'contactprop_c1' => array (
-        0 => 'contact_id',
-        1 => 'type_id',
-        2 => 'rank',
-      ),
-    ),
-    'indexes' => array (
-      'contactprop_idx1' => array (
-        0 => 'contact_id',
-      ),
-      'contactprop_idx2' => array (
-        0 => 'type_id',
-      ),
-    ),
-    'foreign keys' => array (
-      'cvterm' => array (
-        'table' => 'cvterm',
-        'columns' => array (
-          'type_id' => 'cvterm_id',
-        ),
-      ),
-      'contact' => array (
-        'table' => 'contact',
-        'columns' => array (
-          'contact_id' => 'contact_id',
-        ),
-      ),
-    ),
-  );
-  chado_create_custom_table('contactprop', $schema, TRUE);
-} */

legacy/tripal_pub/api/tripal_pub.DEPRECATED.inc

@@ -129,11 +129,11 @@ function tripal_pub_import_publications_by_import_id($import_id, $job_id = NULL)
     "DEPRECATED: %old_function has been replaced with %new_function. Please update your code.",
     array(
       '%old_function'=>'tripal_pub_import_publications_by_import_id',
-      '%new_function' => 'tripal_execute_pub_importer'
+      '%new_function' => 'chado_execute_pub_importer'
     )
   );
 
-  return tripal_execute_pub_importer($import_id, $job_id);
+  return chado_execute_pub_importer($import_id, TRUE, FALSE, $job_id);
 }
 
 /**

legacy/tripal_pub/tripal_pub.install

@@ -16,7 +16,7 @@ function tripal_pub_disable() {
   require_once("tripal_pub.views_default.inc");
   $views = tripal_pub_views_default_views();
   foreach (array_keys($views) as $view_name) {
-    tripal_disable_view($view_name,FALSE,array('suppress_error' => TRUE));
+    tripal_disable_view($view_name, FALSE);
   }
 
 }
@@ -49,17 +49,6 @@ function tripal_pub_requirements($phase) {
 function tripal_pub_install() {
   global $base_path;
 
-  // add loading of the the tripal pub ontology to the job queue
-  $obo_path = '{tripal_pub}/files/tpub.obo';
-  $obo_id = tripal_insert_obo('Tripal Publication', $obo_path);
-  tripal_submit_obo_job(array('obo_id' => $obo_id));
-
-  tripal_pub_add_cvs();
-  tripal_pub_add_cvterms();
-
-  // add the custom tables
-  tripal_pub_add_custom_tables();
-
   // set the default vocabularies
   tripal_set_default_cv('pub', 'type_id', 'tripal_pub');
   tripal_set_default_cv('pubprop', 'type_id', 'tripal_pub');
@@ -126,95 +115,9 @@ function tripal_pub_schema() {
     'primary key' => array('nid'),
   );
 
-  $schema['tripal_pub_import'] = array(
-    'fields' => array(
-      'pub_import_id' => array(
-        'type' => 'serial',
-        'not null' => TRUE
-      ),
-      'name' => array(
-        'type' => 'varchar',
-        'length' => 255,
-        'not null' => TRUE
-      ),
-      'criteria' => array(
-        'type' => 'text',
-        'size' => 'normal',
-        'not null' => TRUE,
-        'description' => 'Contains a serialized PHP array containing the search criteria'
-      ),
-      'disabled'  => array(
-        'type' => 'int',
-        'unsigned' => TRUE,
-        'not NULL' => TRUE,
-        'default' => 0
-      ),
-      'do_contact'  => array(
-        'type' => 'int',
-        'unsigned' => TRUE,
-        'not NULL' => TRUE,
-        'default' => 0
-      ),
-    ),
-    'primary key' => array('pub_import_id'),
-    'indexes' => array(
-      'name' => array('name')
-    ),
-  );
-
   return $schema;
 }
 
-/**
- * Add custom table related to publications
- *  - pubauthor_contact
- *
- * @ingroup tripal_legacy_pub
- */
-// This function was moved to tripal_chado/includes/setup/tripal_chado.setup.inc
-/* function tripal_pub_add_custom_tables() {
-  $schema = array (
-    'table' => 'pubauthor_contact',
-    'fields' => array (
-      'pubauthor_contact_id' => array (
-        'type' => 'serial',
-        'not null' => true,
-      ),
-      'contact_id' => array (
-        'type' => 'int',
-        'not null' => true,
-      ),
-      'pubauthor_id' => array (
-        'type' => 'int',
-        'not null' => true,
-      ),
-    ),
-    'primary key' => array (
-      0 => 'pubauthor_contact_id',
-    ),
-    'unique keys' => array (
-      'pubauthor_contact_c1' => array (
-        0 => 'contact_id',
-        1 => 'pubauthor_id',
-      ),
-    ),
-    'foreign keys' => array (
-      'contact' => array (
-        'table' => 'contact',
-        'columns' => array (
-          'contact_id' => 'contact_id',
-        ),
-      ),
-      'pubauthor' => array (
-        'table' => 'pubauthor',
-        'columns' => array (
-          'pubauthor_id' => 'pubauthor_id',
-        ),
-      ),
-    ),
-  );
-  chado_create_custom_table('pubauthor_contact', $schema, TRUE);
-} */
 
 /**
  * This is the required update for tripal_pub when upgrading from Drupal core API 6.x.

legacy/tripal_pub/tripal_pub.module

@@ -265,31 +265,6 @@ function tripal_pub_permission() {
   );
 }
 
-/**
- * Implements hook_mail().
- *
- * @ingroup tripal_legacy_pub
- */
-function tripal_pub_mail($key, &$message, $params) {
-  $site_name = variable_get('site_name');
-  $language = $message['language'];
-  switch($key) {
-    case 'import_report':
-      $headers = array(
-        'MIME-Version' => '1.0',
-        'Content-Type' => 'text/html; charset=UTF-8; format=flowed',
-        'Content-Transfer-Encoding' => '8Bit',
-        'X-Mailer' => 'Drupal'
-      );
-      foreach ($headers as $key => $value) {
-        $message['headers'][$key] = $value;
-      }
-      $message['subject'] = t('Publication import from !site', array('!site' => $site_name));
-      $message['body'][] = $params['message'];
-      break;
-  }
-}
-
 /**
  * Implementation of hook_form_alter().
  *
@@ -338,7 +313,7 @@ function tripal_pub_form_alter(&$form, &$form_state, $form_id) {
 function tripal_pub_job_describe_args($callback, $args) {
 
   $new_args = array();
-  if ($callback == 'tripal_execute_pub_importer') {
+  if ($callback == 'chado_execute_pub_importer') {
     // get all of the loaders
     $qargs = array(':import_id' => $args[0]);
     $sql = "SELECT * FROM {tripal_pub_import} WHERE pub_import_id = :import_id ";

tripal/api/tripal.jobs.api.inc

@@ -567,4 +567,4 @@ function tripal_execute_job($job_id, $redirect = TRUE) {
   if ($redirect) {
     drupal_goto("admin/tripal/tripal_jobs/view/$job_id");
   }
-}
+}

tripal/api/tripal.notice.api.inc

@@ -31,8 +31,13 @@ define('TRIPAL_DEBUG',7);
 /**
  * Provide better error notice for Tripal.
  *
+ * Please be sure to set the $options array as desired. For example, by default
+ * this function sends all messages to the Drupal watchdog. If a long running
+ * job uses this function and prints status messages you may not want to have
+ * those go to the watchdog as it can dramatically affect performance. 
+ * 
  * If the environment variable 'TRIPAL_DEBUG' is set to 1 then this function
- * will add backtrace information to the message.
+ * will add backtrace information to the message.  
  *
  * @param $type
  *   The catagory to which this message belongs. Can be any string, but the
@@ -60,6 +65,12 @@ define('TRIPAL_DEBUG',7);
  *       display is the command-line
  *     - drupal_set_message: set to TRUE to call drupal_set_message with the 
  *       same error.
+ *     - drupal_set_message:  set to TRUE then send the message to the
+ *       drupal_set_message function.
+ *     - watchdog:  set to FALSE to disable logging to Drupal's watchdog.
+ *     - job: The jobs management object for the job if this function is run 
+ *       as a job. Adding the job object here ensures that any status or error
+ *       messages are also logged with the job.
  *
  * @ingroup tripal_notify_api
  */
@@ -110,13 +121,15 @@ function tripal_report_error($type, $severity, $message, $variables = array(), $
     }
   }
 
-  // Send to watchdog.
-  try {
-    watchdog($type, $message, $variables, $severity);
-  }
-  catch (Exception $e) {
-    print "CRITICAL (TRIPAL): Unable to add error message with watchdog: " . $e->getMessage(). "\n.";
-    $options['print'] = TRUE;
+  // Send to watchdog if the user wants.
+  if (array_key_exists('watchdog', $options) and $options['watchdog'] !== FALSE) {   
+    try {
+      watchdog($type, $message, $variables, $severity);
+    }
+    catch (Exception $e) {
+      print "CRITICAL (TRIPAL): Unable to add error message with watchdog: " . $e->getMessage(). "\n.";
+      $options['print'] = TRUE;
+    }
   }
 
   // Format the message for printing (either to the screen, log or both).
@@ -149,7 +162,10 @@ function tripal_report_error($type, $severity, $message, $variables = array(), $
   if (($severity != TRIPAL_INFO)) {
     tripal_log('[' . strtoupper($type) . '] ' . $print_message . "\n", $severity_string);
   }
-
+  
+  if (array_key_exists('job', $options) and is_a($options['job'], 'TripalJob')) {   
+    $options['job']->logMessage($message, $variables, $severity);
+  }
 }
 
 /**
@@ -231,8 +247,12 @@ function tripal_set_message($message, $importance = TRIPAL_INFO, $options = arra
 }
 
 /**
- * File-based logging for Tripal.
- *
+ * File-based error logging for Tripal.
+ * 
+ * Consider using the tripal_report_error function rather than
+ * calling this function directly, as that function calls this one for non
+ * INFO messages and has greater functionality. 
+ * 
  * @param $message
  *   The message to be logged. Need not contain date/time information.
  * @param $log_type
@@ -240,7 +260,7 @@ function tripal_set_message($message, $importance = TRIPAL_INFO, $options = arra
  *   are supported.
  * @param $options
  *   An array of options where the following keys are supported:
- *     - first_progress_bar: this sohuld be used for the first log call for a
+ *     - first_progress_bar: this should be used for the first log call for a
  *       progress bar.
  *     - is_progress_bar: this option should be used for all but the first print
  *       of a progress bar to allow it all to be printed on the same line

tripal/includes/TripalJob.inc

@@ -595,6 +595,15 @@ class TripalJob {
    * Logging works regardless if the job uses a transaction. If the
    * transaction must be rolled back to to an error the error messages will
    * persist.
+   * 
+   * If a function can be executed by the Tripal job system (and hence the
+   * job object is passed in) then you can directly use this function to
+   * log messages. However, if the function can be run via drush on the 
+   * command-line, consider using the tripal_report_error() function which can
+   * accept a job object as an $option and will print to both the terminal
+   * and to the job object.  If you use the tripal_report_error() be sure
+   * to set the 'watchdog' option only if you need log messages also going 
+   * to the watchdog.
    *
    * @param $message
    *   The message to store in the log. Keep $message translatable by not

tripal/includes/tripal.jobs.inc

@@ -279,12 +279,21 @@ function tripal_jobs_view($job_id) {
   }
 
   // build the links
-  $links  = l('Return to jobs list', "admin/tripal/tripal_jobs/") . ' | ';
-  $links .= l('Re-run this job', "admin/tripal/tripal_jobs/rerun/" . $job->job_id) . ' | ';
+  $items = [];
+  $items[] = l('Return to jobs list', "admin/tripal/tripal_jobs/");
+  $items[] = l('Re-run this job', "admin/tripal/tripal_jobs/rerun/" . $job->job_id);
   if ($job->start_time == 0 and $job->end_time == 0) {
-    $links .= l('Cancel this job', "admin/tripal/tripal_jobs/cancel/" . $job->job_id) . ' | ';
-    $links .= l('Execute this job', "admin/tripal/tripal_jobs/execute/".$job->job_id);
+    $items[] = l('Cancel this job', "admin/tripal/tripal_jobs/cancel/" . $job->job_id);
+    $items[] = l('Execute this job', "admin/tripal/tripal_jobs/execute/".$job->job_id);
   }
+  $links = theme_item_list([    
+    'items' => $items,
+    'title' => '',
+    'type' => 'ul',
+    'attributes' => [
+      'class' => ['action-links'],
+    ],
+  ]);
 
   // make our start and end times more legible
   $job->submit_date = tripal_get_job_submit_date($job);
@@ -320,7 +329,7 @@ function tripal_jobs_view($job_id) {
 
   $content['links'] = array(
     '#type' => 'markup',
-    '#markup' => '<p>' . substr($links, 0, -2) . '</p>',
+    '#markup' => $links,
   );
   $content['job_title'] = array(
     '#type' => 'item',
@@ -362,7 +371,7 @@ function tripal_jobs_view($job_id) {
   );
   $content['log_fset']['job_logs'] = array(
     '#type' => 'markup',
-    '#markup' => '<pre>' . $job->error_msg . '</pre>',
+    '#markup' => '<pre class="tripal-job-logs">' . $job->error_msg . '</pre>',
   );
   return $content;
 }

tripal/theme/css/tripal.css

@@ -103,4 +103,14 @@ div.messages.tripal-site-admin-only {
    stroke: black;
  }
  
- #tripal-entity-type-chart .x.axis text { display: none; }
+ #tripal-entity-type-chart .x.axis text { display: none; }
+ 
+ .tripal-job-logs {
+  height: 500px;
+  overflow: scroll;
+  font-framily: Courier New, monospace;
+}
+
+.item-list .action-links li {
+	list-style-type: none;
+} 

tripal_chado/api/modules/tripal_chado.module.DEPRECATED.api.inc

@@ -1365,7 +1365,7 @@ function tripal_import_pub_by_dbxref($pub_dbxref, $do_contact = false, $do_updat
  * @ingroup tripal_chado_module_DEPRECATED_api
  */
 function tripal_execute_active_pub_importers($report_email = false, $do_update = false) {
-  return chado_execute_active_pub_importers($report_email, $do_update);
+  return chado_execute_active_pub_importers($report_email, TRUE, $do_update);
 }
 
 /**

tripal_chado/api/modules/tripal_chado.pub.api.inc

@@ -331,13 +331,21 @@ function chado_autocomplete_pub($string = '') {
  * @param $do_contact
  *   Set to TRUE if authors should automatically have a contact record added
  *   to Chado.
+ * @param $publish
+ *   Set to TRUE if publications should be published after import.  For Tripal
+ *   v3 this value can be set to the string 'sync' or 'both' in the event that
+ *   the site is in "legacy" mode.  Setting this value to 'sync' will create
+ *   nodes, setting to 'both' will create nodes and entities.  If set to TRUE
+ *   only entities are created.
  * @param $do_update
  *   If set to TRUE then the publication will be updated if it already exists
  *   in the database.
  *
  * @ingroup tripal_pub_api
  */
-function chado_import_pub_by_dbxref($pub_dbxref, $do_contact = FALSE, $do_update = TRUE) {
+function chado_import_pub_by_dbxref($pub_dbxref, $do_contact = FALSE, 
+    $publish = TRUE, $do_update = TRUE) {
+    
   $num_to_retrieve = 1;
   $pager_id = 0;
   $page = 0;
@@ -345,10 +353,21 @@ function chado_import_pub_by_dbxref($pub_dbxref, $do_contact = FALSE, $do_update
   $pub_id = NULL;
 
   module_load_include('inc', 'tripal_chado', 'includes/loaders/tripal_chado.pub_importers');
-
-  print "\nNOTE: Loading of publications is performed using a database transaction. \n" .
-      "If the load fails or is terminated prematurely then the entire set of \n" .
-      "insertions/updates is rolled back and will not be found in the database\n\n";
+  
+  // These are options for the tripal_report_error function. We do not
+  // want to log messages to the watchdog but we do for the job and to
+  // the terminal
+  $message_type = 'pub_import';
+  $message_opts = [
+    'watchdog' == FALSE,
+    'print' => TRUE,
+  ];
+  
+  $message = "Importing of publications is performed using a database transaction. " .
+    "If the load fails or is terminated prematurely then the entire set of " .
+    "deletions is rolled back and will not be found in the database";
+  tripal_report_error($message_type, TRIPAL_INFO, $message, [], $message_opts);
+  
 
   $transaction = db_transaction();
   try {
@@ -370,27 +389,15 @@ function chado_import_pub_by_dbxref($pub_dbxref, $do_contact = FALSE, $do_update
       );
       $remote_db = $criteria['remote_db'];
       $results = tripal_get_remote_pubs($remote_db, $criteria, $num_to_retrieve, $page);
-      $pubs          = $results['pubs'];
-      $search_str    = $results['search_str'];
+      $pubs = $results['pubs'];
+      $search_str = $results['search_str'];
       $total_records = $results['total_records'];
       tripal_pub_add_publications($pubs, $do_contact, $do_update);
     }
 
-    // For backwards compatibility check to see if the legacy pub module
-    // is enabled. If so, then sync the nodes.
-    if (module_exists('tripal_pub')) {
-
-      // Sync the newly added publications with Drupal.
-      print "Syncing publications with Drupal...\n";
-      chado_node_sync_records('pub');
-
-      // If any of the importers wanted to create contacts from the authors 
-      // then sync them.
-      if($do_contact) {
-        print "Syncing contacts with Drupal...\n";
-        chado_node_sync_records('contact');
-      }
-    }
+    // Publish as requested by the caller.
+    _chado_execute_pub_importer_publish($publish, NULL, $message_type, $message_opts);
+    
   }
   catch (Exception $e) {
     $transaction->rollback();
@@ -407,108 +414,212 @@ function chado_import_pub_by_dbxref($pub_dbxref, $do_contact = FALSE, $do_update
  * @param $report_email
  *   A list of email address, separated by commas, that should be notified
  *   once importing has completed.
+ * @param $publish
+ *   Set to TRUE if publications should be published after import.  For Tripal
+ *   v3 this value can be set to the string 'sync' or 'both' in the event that
+ *   the site is in "legacy" mode.  Setting this value to 'sync' will create
+ *   nodes, setting to 'both' will create nodes and entities.  If set to TRUE
+ *   only entities are created.
  * @param $do_update
  *   If set to TRUE then publications that already exist in the Chado database
  *   will be updated, whereas if FALSE only new publications will be added.
  *
  * @ingroup tripal_pub_api
  */
-function chado_execute_active_pub_importers($report_email = FALSE, $do_update = FALSE) {
-  $num_to_retrieve = 100;
-  $page = 0;
+function chado_execute_active_pub_importers($report_email = FALSE, 
+  $publish = TRUE, $do_update = FALSE) {
+
+  $report = [];
+  $report['error'] = [];
+  $report['inserted'] = [];
+  $report['skipped'] = [];
+  $report['updated'] = [];
+    
+  // Get all of the loaders.
+  $args = array();
+  $sql = "SELECT * FROM {tripal_pub_import} WHERE disabled = 0 ";
+  $importers = db_query($sql, $args);
+  $do_contact = FALSE;
+  while ($import = $importers->fetchObject()) {
+    $importer_report = chado_execute_pub_importer($import->pub_import_id, $publish, $do_update);
+    foreach ($importer_report as $action => $pubs) {
+      $report[$action] = array_merge($report[$action], $pubs);
+    }
+  }
 
-  print "\nNOTE: Loading of publications is performed using a database transaction. \n" .
-      "If the load fails or is terminated prematurely then the entire set of \n" .
-      "insertions/updates is rolled back and will not be found in the database\n\n";
+  $site_email = variable_get('site_mail', '');
+  $params = array(
+    'report' => $report
+  );
+  drupal_mail('tripal_chado', 'import_report', $report_email, language_default(), $params, $site_email, TRUE);
+  print "Done.\n";
+}
 
+
+/**
+ * Imports all publications for a given publication import setup.
+ *
+ * @param $import_id
+ *   The ID of the import setup to use
+ * @param $publish
+ *   Set to TRUE if publications should be published after import.  For Tripal
+ *   v3 this value can be set to the string 'sync' or 'both' in the event that
+ *   the site is in "legacy" mode.  Setting this value to 'sync' will create
+ *   nodes, setting to 'both' will create nodes and entities.  If set to TRUE
+ *   only entities are created.
+ * @param $do_update
+ *   If set to TRUE then publications that already exist in the Chado database
+ *   will be updated, whereas if FALSE only new publications will be added.
+ * @param $job
+ *   The jobs management object for the job if this function is run as a job. 
+ *   This argument is added by Tripal during a job run and is not needed if
+ *   this function is run directly.
+ *   
+ * @return
+ *   Returns an array containing the number of publications that were
+ *   inserted, updated, skipped and which had an error during import.
+ *
+ * @ingroup tripal_pub
+ */
+function chado_execute_pub_importer($import_id, $publish = TRUE, 
+  $do_update = FALSE, $job = NULL) {
+  
+  // Holds the list of imported pubs which includes their ID and Citation.
+  $report = [];
+  $report['error'] = [];
+  $report['inserted'] = [];
+  $report['skipped'] = [];
+  $report['updated'] = [];
+  
+  // These are options for the tripal_report_error function. We do not
+  // want to log messages to the watchdog but we do for the job and to
+  // the terminal
+  $message_type = 'pub_import';
+  $message_opts = [
+    'watchdog' == FALSE,
+    'job' => $job,
+    'print' => TRUE,
+  ];
+  
+  $message = "Importing of publications for this importer is performed using a database transaction. " .
+    "If the load fails or is terminated prematurely then the entire set of " .
+    "deletions is rolled back and will not be found in the database";  
+  tripal_report_error($message_type, TRIPAL_INFO, $message, [], $message_opts);
+  
+   
   // start the transaction
   $transaction = db_transaction();
-
+  
   try {
-    // Get all of the loaders.
-    $args = array();
-    $sql = "SELECT * FROM {tripal_pub_import} WHERE disabled = 0 ";
-    $results = db_query($sql, $args);
+    $page = 0;
     $do_contact = FALSE;
-    $reports = array();
-    foreach ($results as $import) {
-      $page = 0;
-      print "Executing importer: '" . $import->name . "'\n";
-      // Keep track if any of the importers want to create contacts from authors.
-      if ($import->do_contact == 1) {
-        $do_contact = TRUE;
-      }
-      $criteria = unserialize($import->criteria);
-      $remote_db = $criteria['remote_db'];
-      do {
-        // Retrieve the pubs for this page. We'll retreive 100 at a time.
-        $results = tripal_get_remote_pubs($remote_db, $criteria, $num_to_retrieve, $page);
-        $pubs = $results['pubs'];
-        $reports[$import->name] = tripal_pub_add_publications($pubs, $import->do_contact, $do_update);
-        $page++;
+    $num_to_retrieve = 100;
+    
+    // get all of the loaders
+    $args = array(':import_id' => $import_id);
+    $sql = "SELECT * FROM {tripal_pub_import} WHERE pub_import_id = :import_id ";
+    $import = db_query($sql, $args)->fetchObject();
+    
+    tripal_report_error($message_type, TRIPAL_INFO, 
+      "Executing Importer: !name.", ['!name' => $import->name], $message_opts);
+    
+    $criteria = unserialize($import->criteria);
+    $remote_db = $criteria['remote_db'];
+    $total_pubs = 0;
+    
+    // Loop until we have a $pubs array that does not have
+    // our requested numer of records.  This means we've hit the end    
+    do {
+      // retrieve the pubs for this page. We'll retreive 100 at a time
+      tripal_report_error($message_type, TRIPAL_INFO,
+        "Querying !remote_db for up to !num pubs that match the criteria.", 
+        ['!num' => $num_to_retrieve, '!remote_db' => $remote_db], $message_opts);
+      $results = tripal_get_remote_pubs($remote_db, $criteria, $num_to_retrieve, $page);
+      $pubs = $results['pubs'];
+      $num_pubs = $results['total_records'];
+      $total_pubs += $num_pubs;
+      tripal_report_error($message_type, TRIPAL_INFO,
+        "Found %num publications.",
+        ['%num' => $num_pubs], $message_opts);
+      
+      $subset_report = tripal_pub_add_publications($pubs, $import->do_contact, $do_update, $job);
+      foreach ($subset_report as $action => $pubs) {
+        $report[$action] = array_merge($report[$action], $pubs);
       }
-      // Continue looping until we have a $pubs array that does not have
-      // our requested numer of records.  This means we've hit the end.
-      while (count($pubs) == $num_to_retrieve);
+      $page++;
     }
-
-    // Sync the newly added publications with Drupal. If the user
-    // requested a report then we don't want to print any syncing information
-    // so pass 'FALSE' to the sync call.
-    // For backwards compatibility check to see if the legacy pub module
-    // is enabled. If so, then sync the nodes.
-    if (module_exists('tripal_pub')) {
-      print "Syncing publications with Drupal...\n";
-      chado_node_sync_records('pub');
+    while (count($pubs) == $num_to_retrieve);
+    
+    // Publish as requested by the caller.
+    _chado_execute_pub_importer_publish($publish, $job, $message_type, $message_opts);
+    
+    if ($job) {
+      $job->setProgress(100);
     }
+  }
+  catch (Exception $e) {
+    $transaction->rollback();
+    watchdog_exception('T_pub_import', $e);
+    tripal_report_error($message_type, TRIPAL_ERROR,
+      "Rolling back database changes... !message", 
+      ['!message' => $e->getMessage()], $message_opts);
+    return FALSE;
+  }
+  
+  tripal_report_error($message_type, TRIPAL_INFO,
+    "Done.", [], $message_opts);
+  
+  return $report;
+}
 
-    // Iterate through each of the reports and generate a final report with HTML
-    // links.
-    $HTML_report = '';
-    if ($report_email) {
-      $HTML_report .= "<html>";
-      global $base_url;
-      foreach ($reports as $importer => $report) {
-        $total = count($report['inserted']);
-        $HTML_report .= "<b>$total new publications from importer: $importer</b><br><ol>\n";
-        foreach ($report['inserted'] as $pub) {
-          $item = $pub['Title'];
-          if (array_key_exists('pub_id', $pub)) {
-            $item = l($pub['Title'], "$base_url/pub/" . $pub['pub_id']);
-          }
-          $HTML_report .= "<li>$item</li>\n";
-        }
-        $HTML_report .= "</ol>\n";
-      }
-      $HTML_report .= "</html>";
-      $site_email = variable_get('site_mail', '');
-      $params = array(
-        'message' => $HTML_report
-      );
-      drupal_mail('tripal_pub', 'import_report', $report_email, language_default(), $params, $site_email, TRUE);
-    }
 
-    // For backwards compatibility check to see if the legacy pub module
-    // is enabled. If so, then sync the nodes.
+/**
+ * A helper function to dermine if imported publications should be published.
+ * 
+ * It supports backwards compatibility with Tripal v2 legacy mode.
+ * 
+ * @param $publish
+ *   Set to TRUE if publications should be published after import.  For Tripal
+ *   v3 this value can be set to the string 'sync' or 'both' in the event that
+ *   the site is in "legacy" mode.  Setting this value to 'sync' will create
+ *   nodes, setting to 'both' will create nodes and entities.  If set to TRUE
+ *   only entities are created.
+ */
+function _chado_execute_pub_importer_publish($publish, $job, $message_type, $message_opts) {
+    
+  // If the user wants to publish then do so.
+  if ($publish === TRUE or $publish === 'both') {
+    
+    // Get the bundle for the Publication content type.
+    $bundle_term = tripal_load_term_entity(['vocabulary' => 'TPUB', 'accession' => '0000002']);
+    $bundle = tripal_load_bundle_entity(['term_id' => $bundle_term->id]);
+    if ($bundle) {
+      tripal_report_error($message_type, TRIPAL_INFO,
+        "Publishing publications with Drupal...", [], $message_opts);
+      chado_publish_records(['bundle_name' => $bundle->name], $job);
+    }
+    // Note: we won't publish contacts as Tripal v2 did because there is
+    // no consisten way to do that. Each site my use a different term for
+    // different contact content types (e.g. all as one 'Contact' type or
+    // specific such as 'Person', 'Organization', etc.).
+  }
+  
+  // For backwords compatibility with legacy module do a sync.
+  if ($publish === 'sync' or $publish === 'both') {
     if (module_exists('tripal_pub')) {
-      // If any of the importers wanted to create contacts from the authors then 
-      // sync them.
-      if($do_contact) {
-        print "Syncing contacts with Drupal...\n";
+      tripal_report_error($message_type, TRIPAL_INFO,
+        "Syncing publications with Drupal...", [], $message_opts);
+      chado_node_sync_records('pub');
+      if($import->do_contact) {
+        tripal_report_error($message_type, TRIPAL_INFO,
+          "Syncing contacts with Drupal...", [], $message_opts);
         chado_node_sync_records('contact');
       }
     }
   }
-  catch (Exception $e) {
-    $transaction->rollback();
-    print "\n"; // make sure we start errors on new line
-    watchdog_exception('T_pub_import', $e);
-    print "FAILED: Rolling back database changes...\n";
-    return;
-  }
-  print "Done.\n";
 }
 
+
 /**
  * Updates publication records.
  *
@@ -528,14 +639,32 @@ function chado_execute_active_pub_importers($report_email = FALSE, $do_update =
  *   The name of the remote database to update.  If this value is provided and
  *   no dbxref then all of the publications currently in the Chado database
  *   for this remote database will be updated.
- *
+ * @param $publish
+ *   Set to TRUE if publications should be published after import.  For Tripal
+ *   v3 this value can be set to the string 'sync' or 'both' in the event that
+ *   the site is in "legacy" mode.  Setting this value to 'sync' will create
+ *   nodes, setting to 'both' will create nodes and entities.  If set to TRUE
+ *   only entities are created.
+
  * @ingroup tripal_pub_api
  */
-function chado_reimport_publications($do_contact = FALSE, $dbxref = NULL, $db = NULL) {
-
-  print "\nNOTE: Loading of publications is performed using a database transaction. \n" .
-      "If the load fails or is terminated prematurely then the entire set of \n" .
-      "insertions/updates is rolled back and will not be found in the database\n\n";
+function chado_reimport_publications($do_contact = FALSE, $dbxref = NULL, 
+    $db = NULL, $publish = TRUE) {
+
+  // These are options for the tripal_report_error function. We do not
+  // want to log messages to the watchdog but we do for the job and to
+  // the terminal
+  $message_type = 'pub_import';
+  $message_opts = [
+    'watchdog' == FALSE,
+    'print' => TRUE,
+  ];
+  
+  $message = "Importing of publications for this importer is performed using a database transaction. " .
+    "If the load fails or is terminated prematurely then the entire set of " .
+    "deletions is rolled back and will not be found in the database";
+  tripal_report_error($message_type, TRIPAL_INFO, $message, [], $message_opts);
+  
   $transaction = db_transaction();
   try {
 
@@ -543,10 +672,10 @@ function chado_reimport_publications($do_contact = FALSE, $dbxref = NULL, $db =
     // databases.
     $sql = "
       SELECT DB.name as db_name, DBX.accession
-      FROM pub P
-        INNER JOIN pub_dbxref PDBX ON P.pub_id = PDBX.pub_id
-        INNER JOIN dbxref DBX      ON DBX.dbxref_id = PDBX.dbxref_id
-        INNER JOIN db DB           ON DB.db_id = DBX.db_id
+      FROM {pub} P
+        INNER JOIN {pub_dbxref} PDBX ON P.pub_id = PDBX.pub_id
+        INNER JOIN {dbxref} DBX      ON DBX.dbxref_id = PDBX.dbxref_id
+        INNER JOIN {db} DB           ON DB.db_id = DBX.db_id
     ";
     $args = array();
     if ($dbxref and preg_match('/^(.*?):(.*?)$/', $dbxref, $matches)) {
@@ -591,32 +720,22 @@ function chado_reimport_publications($do_contact = FALSE, $dbxref = NULL, $db =
           ),
         ),
       );
-      $pubs = tripal_get_remote_pubs($remote_db, $search, 1, 0);
+      $qresults = tripal_get_remote_pubs($remote_db, $search, 1, 0);
+      $pubs = $qresults['pubs'];      
       tripal_pub_add_publications($pubs, $do_contact, TRUE);
 
       $i++;
     }
 
-    // For backwards compatibility check to see if the legacy pub module
-    // is enabled. If so, then sync the nodes.
-    if (module_exists('tripal_pub')) {
-      // Sync the newly added publications with Drupal.
-      print "Syncing publications with Drupal...\n";
-      chado_node_sync_records('pub');
-
-      // If the caller wants to create contacts then we should sync them.
-      if ($do_contact) {
-        print "Syncing contacts with Drupal...\n";
-        chado_node_sync_records('contact');
-      }
-    }
+    // Publish as requested by the caller.
+    _chado_execute_pub_importer_publish($publish, NULL, $message_type, $message_opts);
   }
   catch (Exception $e) {
     $transaction->rollback();
-    print "\n"; // make sure we start errors on new line
     watchdog_exception('T_pub_import', $e);
-    print "FAILED: Rolling back database changes...\n";
-    return;
+    tripal_report_error($message_type, TRIPAL_ERROR,
+      "Rolling back database changes... !message",
+      ['!message' => $e->getMessage()], $message_opts);return;
   }
   print "Done.\n";
 }

tripal_chado/api/tripal_chado.api.inc

@@ -26,10 +26,10 @@
  * @param $values
  *   A key/value associative array that supports the following keys:
  *   - bundle_name:  The name of the the TripalBundle (e.g. bio_data-12345).
- * @param $job_id
- *   (Optional) The numeric job ID as provided by the Tripal jobs system. There
- *   is no need to specify this argument if this function is being called
- *   outside of the jobs systems.
+ * @param $job
+ *   The jobs management object for the job if this function is run as a job. 
+ *   This argument is added by Tripal during a job run and is not needed if
+ *   this function is run directly.
  *
  * @return boolean
  *   TRUE if all of the records of the given bundle type were published, and
@@ -37,23 +37,27 @@
  *
  * @ingroup tripal_chado_api
  */
-function chado_publish_records($values, $job_id = NULL) {
-
+function chado_publish_records($values, $job = NULL) {
+ 
   // Used for adding runtime to the progress report.
   $started_at = microtime(true);
 
-  // We want the job object in order to report progress.
-  if (is_object($job_id)) {
-    $job = $job_id;
-  }
-  if (is_numeric($job_id)) {
+  // We want the job object in order to report progress with the job.
+  if (is_numeric($job)) {
+    $job_id = $job;
     $job = new TripalJob();
     $job->load($job_id);
   }
-  $report_progress = FALSE;
-  if (is_object($job)) {
-    $report_progress = TRUE;
-  }
+  
+  // These are options for the tripal_report_error function. We do not
+  // want to log messages to the watchdog but we do for the job and to
+  // the terminal
+  $message_type = 'publish_records';
+  $message_opts = [
+    'watchdog' == FALSE,
+    'job' => $job,
+    'print' => TRUE,
+  ];
 
   // Start an array for caching objects to save performance.
   $cache = array();
@@ -62,7 +66,7 @@ function chado_publish_records($values, $job_id = NULL) {
   if (!array_key_exists('bundle_name', $values) or !$values['bundle_name']) {
     tripal_report_error('tripal_chado', TRIPAL_ERROR,
       "Could not publish record: @error",
-      array('@error' => 'The bundle name must be provided'));
+     ['@error' => 'The bundle name must be provided'], $message_opts);
     return FALSE;
   }
 
@@ -81,9 +85,9 @@ function chado_publish_records($values, $job_id = NULL) {
   $bundle = tripal_load_bundle_entity(array('name' => $bundle_name));
   $cache['bundle'] = $bundle;
   if (!$bundle) {
-    tripal_report_error('tripal_chado', TRIPAL_ERROR,
-        "Unknown bundle. Could not publish record: @error",
-        array('@error' => 'The bundle name must be provided'));
+    tripal_report_error($message_type, TRIPAL_ERROR,
+      "Unknown bundle. Could not publish record: @error",
+      ['@error' => 'The bundle name must be provided'], $message_opts);
     return FALSE;
   }
   $chado_entity_table = chado_get_bundle_entity_table($bundle);
@@ -95,13 +99,14 @@ function chado_publish_records($values, $job_id = NULL) {
     ->execute()
     ->fetchObject();
   if(!$chado_bundle) {
-    tripal_report_error('tripal_chado', TRIPAL_ERROR,
-        "Cannot find mapping of bundle to Chado tables. Could not publish record.");
+    tripal_report_error($message_type, TRIPAL_ERROR,
+      "Cannot find mapping of bundle to Chado tables. Could not publish record.",
+      [], $message_opts);
     return FALSE;
   }
 
   // Load the term for use in setting the alias for each entity created.
-  $term = entity_load('TripalTerm', array('id' => $entity->term_id));
+  $term = entity_load('TripalTerm', array('id' => $bundle->term_id));
   $cache['term'] = $term;
 
   $table = $chado_bundle->data_table;
@@ -202,13 +207,10 @@ function chado_publish_records($values, $job_id = NULL) {
   $sql = "SELECT count(*) as num_records " . $from . $where;
   $result = chado_query($sql, $args);
   $count = $result->fetchField();
-  print "\nThere are $count records to publish.\n";
-
-  print "\nNOTE: publishing records is performed using database transactions. If the job fails\n" .
-          "or is terminated prematurely then the current set of $chunk_size is rolled back with\n" .
-          "no changes to the database. Simply re-run the publishing job to publish any remaining\n".
-          "content after fixing the issue that caused the job to fail.\n\n" .
-          "Also, the following progress only updates every $chunk_size records.\n";
+  
+  tripal_report_error($message_type, TRIPAL_INFO,
+    "There are !count records to publish.",
+    ['!count' => $count], $message_opts);
 
   // Perform the query.
   $sql = $select . $from . $where . ' LIMIT '.$chunk_size;
@@ -220,7 +222,10 @@ function chado_publish_records($values, $job_id = NULL) {
 
     // Update the job status every chunk start.
     // Because this is outside of hte transaction, we can update the admin through the jobs UI.
-    $complete = ($total_published / $count) * 33.33333333;
+    $complete = 0;
+    if ($count > 0) {
+      $complete = ($total_published / $count) * 33.33333333;
+    }
     if ($report_progress) { $job->setProgress(intval($complete * 3)); }
     if ($total_published === 0) {
       printf("%d of %d records. (%0.2f%%) Memory: %s bytes.\r",
@@ -290,7 +295,7 @@ function chado_publish_records($values, $job_id = NULL) {
     catch (Exception $e) {
       $transaction->rollback();
       $error = $e->getMessage();
-      tripal_report_error('tripal_chado', TRIPAL_ERROR, "Could not publish record: @error", array('@error' => $error));
+      tripal_report_error($message_type, TRIPAL_ERROR, "Could not publish record: @error", array('@error' => $error));
       drupal_set_message('Failed publishing record. See recent logs for more details.', 'error');
       return FALSE;
     }
@@ -304,7 +309,10 @@ function chado_publish_records($values, $job_id = NULL) {
     unset($transaction);
   }
 
-  drupal_set_message("Succesfully published $total_published " . $bundle->label . " record(s).");
+  tripal_report_error($message_type, TRIPAL_INFO,
+    "Succesfully published %count %type record(s).",
+    ['%count' => $total_published, '%type' => $bundle->label], $message_opts);
+  
   return TRUE;
 }
 

tripal_chado/api/tripal_chado.property.api.inc

@@ -335,7 +335,7 @@ function chado_update_property($record, $property, $options = array()) {
 
   // First see if the property is missing (we can't update a missing property.
   $prop = chado_get_property($record, $property);
-  if (count($prop) == 0) {
+  if (!is_array($prop) or count($prop) == 0) {
     if ($insert_if_missing) {
       return chado_insert_property($record, $property);
     }

tripal_chado/files/tpub.obo

@@ -1266,7 +1266,7 @@ subset: MeSH_Publication_Type
 
 [Term]
 id: TPUB:0000231
-name: Video Audio Media
+name: Video-Audio Media
 def: Used with articles which include video files or clips, or for articles which are entirely video.
 is_a: TPUB:0000015 ! Publication Type
 subset: MeSH_Publication_Type

tripal_chado/includes/loaders/tripal_chado.pub_importers.inc

@@ -367,7 +367,7 @@ function tripal_pub_importer_setup_form($form, &$form_state = NULL, $pub_import_
   );
   $dbs = chado_select_record('db', array('*'), $values);
   foreach ($dbs as $index => $db) {
-    $remote_dbs[$db->name] = $db->description;
+    $remote_dbs[$db->name] = $db->description ? $db->description : $db->name;
   };
   // use PubMed as the default
   if (!$remote_db) {
@@ -379,7 +379,7 @@ function tripal_pub_importer_setup_form($form, &$form_state = NULL, $pub_import_
    */
   unset($remote_dbs['AGL']);
   $form['themed_element']['remote_db'] = array(
-    '#title' => t('Remote Database'),
+    '#title' => t('Source'),
     '#type' => 'select',
     '#options' => $remote_dbs,
     '#default_value' => $remote_db,
@@ -739,7 +739,7 @@ function tripal_pub_importer_setup_form_submit($form, &$form_state) {
         // if the user wants to do the import now then do it (may time out
         // for long jobs)
         if ($form_state['values']['op'] == 'Save & Import Now') {
-          tripal_execute_pub_importer($record['pub_import_id']);
+          chado_execute_pub_importer($record['pub_import_id']);
         }
         drupal_goto('admin/tripal/loaders/pub');
       }
@@ -844,11 +844,11 @@ function tripal_pub_importer_submit_job($import_id) {
   $sql = "SELECT * FROM {tripal_pub_import} WHERE pub_import_id = :import_id ";
   $import = db_query($sql, $args)->fetchObject();
 
-  $args = array($import_id);
+  $args = array($import_id, TRUE, FALSE);
   $includes = array();
   $includes[] = module_load_include('inc', 'tripal_chado', 'includes/loaders/tripal_chado.pub_importers');
   tripal_add_job("Import publications $import->name", 'tripal_chado',
-    'tripal_execute_pub_importer', $args, $user->uid, 10, $includes);
+    'chado_execute_pub_importer', $args, $user->uid, 10, $includes);
 
   drupal_goto('admin/tripal/loaders/pub');
 }
@@ -883,6 +883,10 @@ function tripal_pub_importer_delete($import_id) {
  * @param $update
  *   If set to TRUE then publications that already exist in the Chado database
  *   will be updated, whereas if FALSE only new publications will be added
+ * @param $job
+ *   The jobs management object for the job if this function is run as a job. 
+ *   This argument is added by Tripal during a job run and is not needed if
+ *   this function is run directly.
  *
  * @return
  *   Returns an array containing the number of publications that were
@@ -890,11 +894,23 @@ function tripal_pub_importer_delete($import_id) {
  *
  * @ingroup tripal_pub
  */
-function tripal_pub_add_publications($pubs, $do_contact, $update = FALSE) {
-  $report = array();
-  $report['error'] = 0;
-  $report['inserted'] = array();
-  $report['skipped'] = array();
+function tripal_pub_add_publications($pubs, $do_contact, $update = FALSE, $job = NULL) {
+  
+  // These are options for the tripal_report_error function. We do not
+  // want to log messages to the watchdog but we do for the job and to
+  // the terminal
+  $message_type = 'pub_import';
+  $message_opts = [
+    'watchdog' == TRUE,
+    'job' => $job,
+    'print' => TRUE,
+  ];
+  
+  $report = [];
+  $report['error'] = [];
+  $report['inserted'] = [];
+  $report['skipped'] = [];
+  $report['updated'] = [];
   $total_pubs = count($pubs);
 
   // iterate through the publications and add each one
@@ -905,7 +921,7 @@ function tripal_pub_add_publications($pubs, $do_contact, $update = FALSE) {
 
     // add the publication to Chado
     $action = '';
-    $pub_id = tripal_pub_add_publication($pub, $action, $do_contact, $update);
+    $pub_id = tripal_pub_add_publication($pub, $action, $do_contact, $update, $job);
     if ($pub_id){
       // add the publication cross reference (e.g. to PubMed)
       if ($pub_id and $pub['Publication Dbxref']) {
@@ -915,11 +931,9 @@ function tripal_pub_add_publications($pubs, $do_contact, $update = FALSE) {
           $dbxref['accession'] = $matches[2];
         }
         else {
-          tripal_report_error(
-            'tripal_pub',
-            TRIPAL_ERROR,
+          tripal_report_error($message_type, TRIPAL_ERROR,
             'Unable to extract the dbxref to be associated with the publication (pub ID=@pub_id) from @dbxref. This reference should be [database-name]:[accession]',
-            array('@pub_id' => $pub_id, '@dbxref' => $pub['Publication Dbxref'])
+            array('@pub_id' => $pub_id, '@dbxref' => $pub['Publication Dbxref'], $message_opts)
           );
         }
         $pub_dbxref = tripal_associate_dbxref('pub', $pub_id, $dbxref);
@@ -929,26 +943,27 @@ function tripal_pub_add_publications($pubs, $do_contact, $update = FALSE) {
 
     switch ($action) {
       case 'error':
-        $report['error']++;
+        $report['error'][] = $pub['Citation'];
         break;
       case 'inserted':
-        $report['inserted'][] = $pub;
+        $report['inserted'][] = $pub['Citation'];
         break;
       case 'updated':
-        $report['updated'][] = $pub;
+        $report['updated'][] = $pub['Citation'];
         break;
       case 'skipped':
-        $report['skipped'][] = $pub;
+        $report['skipped'][] = $pub['Citation'];
         break;
     }
     $i++;
   }
-  print "\n";
   return $report;
 }
 
 /**
- * Adds a new publication to the Chado, along with all properties and
+ * Adds a new publication to Chado.
+ * 
+ * In addition, all properties and
  * database cross-references. If the publication does not already exist
  * in Chado then it is added.  If it does exist nothing is done.  If
  * the $update parameter is TRUE then the publication is updated if it exists.
@@ -959,62 +974,77 @@ function tripal_pub_add_publications($pubs, $do_contact, $update = FALSE) {
  *   This variable will get set to a text value indicating the action that was
  *   performed. The values include 'skipped', 'inserted', 'updated' or 'error'.
  * @param $do_contact
- *   Optional. Set to TRUE if a contact entry should be added to the Chado contact table
- *   for authors of the publication.
+ *   Optional. Set to TRUE if a contact entry should be added to the Chado 
+ *   contact table for authors of the publication.
  * @param $update_if_exists
- *   Optional.  If the publication already exists then this function will return
- *   without adding a new publication.  However, set this value to TRUE to force
- *   the function to pudate the publication using the $pub_details that are provided.
- *
+ *   Optional.  If the publication already exists then this function will 
+ *   return without adding a new publication.  However, set this value to 
+ *   TRUE to force the function to pudate the publication using the 
+ *   $pub_details that are provided.   
+ * @param $job
+ *   The jobs management object for the job if this function is run as a job. 
+ *   This argument is added by Tripal during a job run and is not needed if
+ *   this function is run directly.
  * @return
- *   If the publication already exists, is inserted or updated then the publication
- *   ID is returned, otherwise FALSE is returned. If the publication already exists
- *   and $update_if_exists is not TRUE then the $action variable is set to 'skipped'.
- *   If the publication already exists and $update_if_exists is TRUE and if the update
- *   was successful then $action is set to 'updated'.  Otherwise on successful insert
- *   the $action variable is set to 'inserted'.  If the function failes then the
+ *   If the publication already exists, is inserted or updated then the 
+ *   publication ID is returned, otherwise FALSE is returned. If the 
+ *   publication already exists and $update_if_exists is not TRUE then the 
+ *   $action variable is set to 'skipped'. If the publication already exists 
+ *   and $update_if_exists is TRUE and if the update was successful then 
+ *   $action is set to 'updated'.  Otherwise on successful insert the 
+ *   $action variable is set to 'inserted'.  If the function failes then the
  *   $action variable is set to 'error'
  *
  * @ingroup tripal_pub
  */
-function tripal_pub_add_publication($pub_details, &$action, $do_contact = FALSE, $update_if_exists = FALSE) {
+function tripal_pub_add_publication($pub_details, &$action, $do_contact = FALSE, $update_if_exists = FALSE, $job = NULL) {
   $pub_id = 0;
+  
+  // These are options for the tripal_report_error function. We do not
+  // want to log messages to the watchdog except for errors and to the job and 
+  // to the terminal
+  $message_type = 'pub_import';
+  $message_opts = [
+    'watchdog' == FALSE,
+    'job' => $job,
+    'print' => TRUE,
+  ];
+  $error_opts = [
+    'watchdog' == TRUE,
+    'job' => $job,
+    'print' => TRUE,
+  ];
 
   if (!is_array($pub_details)) {
     return FALSE;
   }
 
-  // before proceeding check to see if the publication already exists. If there is only one match
-  // and the $update_if_exists is NOT set then return FALSE
+  // Before proceeding check to see if the publication already exists. If there 
+  // is only one match and the $update_if_exists is NOT set then return FALSE.
   $pub_ids = chado_publication_exists($pub_details);
 
   if(count($pub_ids) == 1 and !$update_if_exists) {
-    tripal_report_error('tripal_pub', TRIPAL_NOTICE,
-     "There is a publication that is a duplicate of this publication. Cannot continue. It either ".
-     "has a matching Dbxref (e.g. PubMed ID), a non-unique citation or matches on the unique  " .
-     "constraint set by the Tripal publication module configuration page. \nCitation: %title %dbxref.\nMatching Pub id: %ids",
-       array(
-        '%title' => $pub_details['Citation'],
-        '%dbxref' => $pub_details['Publication Dbxref'],
-        '%ids' => implode(",", $pub_ids),
-       )
+    tripal_report_error($message_type, TRIPAL_NOTICE,
+      "The following publication already exists on this site:  %title %dbxref (Matching Pub id: %ids). Skipping.",
+      ['%title' => $pub_details['Citation'],
+       '%dbxref' => $pub_details['Publication Dbxref'],
+       '%ids' => implode(",", $pub_ids)],
+      $message_opts
     );
     $action = 'skipped';
     return FALSE;
   }
-  // if we have more than one matching pub then return an error as we don't know which to update even if
-  // update_if_exists is set to TRUE
+  
+  // If we have more than one matching pub then return an error as we don't 
+  // know which to update even if update_if_exists is set to TRUE.
   if(count($pub_ids) > 1) {
-    tripal_report_error('tripal_pub', TRIPAL_NOTICE,
-      "There are %num publications that are duplicates of this publication. They either " .
-      "have a matching Dbxref (e.g. PubMed ID) or match on the unique constraint set by the Tripal publication module ".
-      "configuration page.  \nCitation: %title %dbxref.\nMatching Pub ids: %ids",
-       array(
-         '%num' => count($pub_ids),
-         '%title' => $pub_details['Citation'],
-         '%dbxref' => $pub_details['Publication Dbxref'],
-        '%ids' => implode(",", $pub_ids),
-       )
+    tripal_report_error($message_type, TRIPAL_NOTICE,
+      "The following publication exists %num times on this site:  %title %dbxref (Matching Pub id: %ids). Skipping.",
+       ['%num' => count($pub_ids),
+        '%title' => $pub_details['Citation'],
+        '%dbxref' => $pub_details['Publication Dbxref'],
+        '%ids' => implode(",", $pub_ids)],
+      $message_opts
     );
     $action = 'skipped';
     return FALSE;
@@ -1023,7 +1053,7 @@ function tripal_pub_add_publication($pub_details, &$action, $do_contact = FALSE,
     $pub_id = $pub_ids[0];
   }
 
-  // get the publication type (use the first publication type)
+  // Get the publication type (use the first publication type).
   if (array_key_exists('Publication Type', $pub_details)) {
     $pub_type = '';
     if(is_array($pub_details['Publication Type'])) {
@@ -1041,19 +1071,21 @@ function tripal_pub_add_publication($pub_details, &$action, $do_contact = FALSE,
     $pub_type = chado_get_cvterm($identifiers);
   }
   else {
-    tripal_report_error('tripal_pub', TRIPAL_ERROR,
-      "The Publication Type is a required property but is missing", array());
+    tripal_report_error($message_type, TRIPAL_ERROR,
+      "The Publication Type is a required property but is missing", [], $error_opts);
     $action = 'error';
     return FALSE;
   }
   if (!$pub_type) {
-    tripal_report_error('tripal_pub', TRIPAL_ERROR, "Cannot find publication type: '%type'",
-      array('%type' => $pub_details['Publication Type'][0]));
+    tripal_report_error($message_type, TRIPAL_ERROR, 
+      "Cannot find publication type: '%type'",
+      ['%type' => $pub_details['Publication Type'][0]], $error_opts);
     $action = 'error';
     return FALSE;
   }
 
-  // the series name field in the pub table is only 255 characters, so we should trim just in case
+  // The series name field in the pub table is only 255 characters, so we 
+  // should trim just in case.
   $series_name = '';
   if (array_key_exists('Series_Name', $pub_details)) {
     $series_name = substr($pub_details['Series Name'], 0, 255);
@@ -1062,7 +1094,7 @@ function tripal_pub_add_publication($pub_details, &$action, $do_contact = FALSE,
     $series_name = substr($pub_details['Journal Name'], 0, 255);
   }
 
-  // build the values array for inserting or updating
+  // Build the values array for inserting or updating.
   $values = array(
     'title'       => $pub_details['Title'],
     'volume'      => (isset($pub_details['Volume'])) ? $pub_details['Volume'] : '',
@@ -1074,13 +1106,14 @@ function tripal_pub_add_publication($pub_details, &$action, $do_contact = FALSE,
     'type_id'     => $pub_type->cvterm_id,
   );
 
-  // if there is no pub_id then we need to do an insert.
+  // If there is no pub_id then we need to do an insert.
   if (!$pub_id) {
     $options = array('statement_name' => 'ins_pub_tivoseispypaunty');
     $pub = chado_insert_record('pub', $values, $options);
     if (!$pub) {
-      tripal_report_error('tripal_pub', TRIPAL_ERROR, "Cannot insert the publication with title: %title",
-      array('%title' => $pub_details['Title']));
+      tripal_report_error($message_type, TRIPAL_ERROR, 
+        "Cannot insert the publication with title: %title",
+        ['%title' => $pub_details['Title']], $error_opts);
       $action = 'error';
       return FALSE;
     }
@@ -1088,22 +1121,24 @@ function tripal_pub_add_publication($pub_details, &$action, $do_contact = FALSE,
     $action = 'inserted';
   }
 
-  // if there is a pub_id and we've been told to update, then do the update
-  if ($pub_id and $update_if_exists) {
+  // If there is a pub_id and we've been told to update, then do the update.
+  else if ($pub_id and $update_if_exists) {
     $match = array('pub_id' => $pub_id);
     $options = array('statement_name' => 'up_pub_tivoseispypaunty');
     $success = chado_update_record('pub', $match, $values, $options);
     if (!$success) {
-      tripal_report_error('tripal_pub', TRIPAL_ERROR, "Cannot update the publication with title: %title",
-      array('%title' => $pub_details['Title']));
+      tripal_report_error($message_type, TRIPAL_ERROR, 
+        "Cannot update the publication with title: %title",
+        ['%title' => $pub_details['Title']], $error_opts);
       $action = 'error';
       return FALSE;
     }
     $action = 'updated';
   }
 
-  // before we add any new properties we need to remove those that are there if this
-  // is an update.  The only thing we don't want to remove are the 'Publication Dbxref'
+  // Before we add any new properties we need to remove those that are there 
+  // if this is an update.  The only thing we don't want to remove are the 
+  // 'Publication Dbxref'.
   if ($update_if_exists) {
     $sql = "
       DELETE FROM {pubprop}
@@ -1118,14 +1153,22 @@ function tripal_pub_add_publication($pub_details, &$action, $do_contact = FALSE,
     chado_query($sql, array(':pub_id' => $pub_id));
   }
 
-  // iterate through the properties and add them
+  // Iterate through the properties and add them.
   foreach ($pub_details as $key => $value) {
-    // the pub_details may have the raw search data (e.g. in XML from PubMed.  We'll irgnore this for now
+    
+    // The pub_details may have the raw search data (e.g. in XML from PubMed. 
+    // We'll irgnore this for now.
     if($key == 'raw') {
       continue;
     }
+    
+    // Since we're not updating the 'Publication Dbxref' on an update
+    // skip this property.
+    if ($update_if_exists and $key == 'Publication Dbxref') {
+      continue;
+    }
 
-    // get the cvterm by name
+    // Get the cvterm by name.
     $identifiers = array(
       'name' => $key,
       'cv_id' => array(
@@ -1134,8 +1177,7 @@ function tripal_pub_add_publication($pub_details, &$action, $do_contact = FALSE,
     );
     $cvterm = chado_get_cvterm($identifiers);
 
-    // if we could not find the cvterm by name then try by synonym
-    //$cvterm = chado_get_cvterm(array('name' => $key, 'cv_id' => array('name' => 'tripal_pub')));
+    // If we could not find the cvterm by name then try by synonym.
     if (!$cvterm) {
       $identifiers = array(
         'synonym' => array(
@@ -1146,11 +1188,12 @@ function tripal_pub_add_publication($pub_details, &$action, $do_contact = FALSE,
       $cvterm = chado_get_cvterm($identifiers);
     }
     if (!$cvterm) {
-      tripal_report_error('tripal_pub', TRIPAL_ERROR, "Cannot find term: '%prop'. Skipping.", array('%prop' => $key));
+      tripal_report_error($message_type, TRIPAL_ERROR, 
+        "Cannot find term: '%prop'. Skipping.", ['%prop' => $key], $error_opts);
       continue;
     }
 
-    // skip details that won't be stored as properties
+    // Skip details that won't be stored as properties.
     if ($key == 'Author List') {
       tripal_pub_add_authors($pub_id, $value, $do_contact);
       continue;
@@ -1163,8 +1206,9 @@ function tripal_pub_add_publication($pub_details, &$action, $do_contact = FALSE,
     $success = 0;
     if (is_array($value)) {
       foreach ($value as $subkey => $subvalue) {
-        // if the key is an integer then this array is a simple list and
-        // we will insert using the primary key. Otheriwse, use the new key
+        
+        // If the key is an integer then this array is a simple list and
+        // we will insert using the primary key. Otheriwse, use the new key.
         if(is_int($subkey)) {
           $success = chado_insert_property(
             array('table' => 'pub', 'id' => $pub_id),
@@ -1187,8 +1231,9 @@ function tripal_pub_add_publication($pub_details, &$action, $do_contact = FALSE,
       );
     }
     if (!$success) {
-      tripal_report_error('tripal_pub', TRIPAL_ERROR, "Cannot add property '%prop' to publication. Skipping.",
-      array('%prop' => $key));
+      tripal_report_error($message_type, TRIPAL_ERROR, 
+        "Cannot add property '%prop' to pubprop table. Skipping.",
+        ['%prop' => $key], $error_opts);
       continue;
     }
   }
@@ -1483,85 +1528,7 @@ function tripal_pub_get_publication_array($pub_id, $skip_existing = TRUE) {
 
 
 
-/**
- * Imports all publications for a given publication import setup.
- *
- * @param $import_id
- *   The ID of the import setup to use
- * @param $job_id
- *   The jobs management job_id for the job if this function is run as a job.
- *
- * @ingroup tripal_pub
- */
-function tripal_execute_pub_importer($import_id, $job_id = NULL) {
-  print "\nNOTE: Loading of publications is performed using a database transaction. \n" .
-      "If the load fails or is terminated prematurely then the entire set of \n" .
-      "insertions/updates is rolled back and will not be found in the database\n\n";
-
-  // start the transaction
-  $transaction = db_transaction();
-
-  try {
-    $page = 0;
-    $do_contact = FALSE;
-    $num_to_retrieve = 100;
-
-    // get all of the loaders
-    $args = array(':import_id' => $import_id);
-    $sql = "SELECT * FROM {tripal_pub_import} WHERE pub_import_id = :import_id ";
-    $import = db_query($sql, $args)->fetchObject();
-
-    print "Executing Importer: '" . $import->name . "'\n";
-
-    $criteria = unserialize($import->criteria);
-    $remote_db = $criteria['remote_db'];
-    $total_pubs = 0;
-    do {
-      // retrieve the pubs for this page. We'll retreive 100 at a time
-      $results  = tripal_get_remote_pubs($remote_db, $criteria, $num_to_retrieve, $page);
-      $pubs     = $results['pubs'];
-      $num_pubs = $rseults['total_records'];
-      $total_pubs += $num_pubs;
-      tripal_pub_add_publications($pubs, $import->do_contact);
-      $page++;
-    }
-    // continue looping until we have a $pubs array that does not have
-    // our requested numer of records.  This means we've hit the end
-    while (count($pubs) == $num_to_retrieve);
-
-    // For backwards compatibility check to see if the legacy pub module
-    // is enabled. If so, then sync the nodes.
-    if (module_exists('tripal_pub')) {
-      // sync the newly added publications with Drupal. If the user
-      // requested a report then we don't want to print any syncing information
-      // so pass 'FALSE' to the sync call
-      print "Syncing publications with Drupal...\n";
-      chado_node_sync_records('pub');
-
-      // if any of the importers wanted to create contacts from the authors then sync them
-      if($import->do_contact) {
-        print "Syncing contacts with Drupal...\n";
-        chado_node_sync_records('contact');
-      }
-    }
-    // For Tripal v3 we want to publish the new publications
-    else {
-      $bundle = tripal_load_bundle_entity(['label' => 'Publication']);
-      if ($bundle) {
-        chado_publish_records(['bundle_name' => $bundle->name], $job_id);
-      }
-    }
-    tripal_set_job_progress($job_id, '100');
-  }
-  catch (Exception $e) {
-    $transaction->rollback();
-    print "\n"; // make sure we start errors on new line
-    watchdog_exception('T_pub_import', $e);
-    print "FAILED: Rolling back database changes...\n";
-    return;
-  }
-  print "Done.\n";
-}
+
 
 /**
  * This function is used to perfom a query using one of the supported databases

tripal_chado/tripal_chado.drush.inc

@@ -84,36 +84,42 @@ function tripal_chado_drush_command() {
     ),
   );
 
+  //
   // Drush commands for publications
+  //
   $items['trp-import-pubs'] = array(
     'description' => dt('Imports publications from remote databases using saved configuration settings.'),
     'options' => array(
-      'create_contacts' => dt('provide this option to create or update contacts for authors. By default contacts are not created or updated.'),
+      'create_contacts' => dt('Create or update contacts for authors. This option is only available with the --dxref option because creation of contacts is otherwise set for each importer.'),
       'dbxref' => dt('An accession number for a publication from a remote database (e.g. PMID:23582642).'),
-      'report' => dt("Set to the email address of the recipient who should receive an HTML report of the publications that have been added."),
+      'report' => dt("Set to the email address of the recipient who should receive an HTML report of the publications that have been added. Not applicable with the --dbxref option."),
       'update' => dt("Set to 'Y' to update existing pubs.  By default only new pubs are inserted."),
+      'publish' => dt("Set to 'Y' to publish publications after import into Chado. (default: Y)."),
+      'sync' => dt("For Tripal v3 legacy mode only. Set to 'Y' to sync publications after import into Chado. This will create legacy \"nodes\" rather than the newer Tripal v3 entities."),
       'username' => array(
         'description' => dt('The Drupal user name for which the job should be run.  The permissions for this user will be used.'),
       ),
     ),
     'examples' => array(
-      'Standard example' => 'drush tripal-pubs-import',
-      'Standard example' => 'drush -l http://[site url] tripal-pubs-import --report=[email]. Where [site url] is the URL of the website and [email] is the email address of the recipient to receive the HTML report',
-      'Import single publication' => 'drush tripal-pub-import --dbxref=PMID:23582642',
+      'Standard example' => 'drush trp-import-pubs --username=[username] --report=[email]. Where [site url] is the URL of the website and [email] is the email address of the recipient to receive the HTML report',
+      'Import single publication' => 'drush tripal-pub-import --username=[username] --dbxref=PMID:23582642',
     ),
   );
   $items['trp-update-pubs'] = array(
     'description' => dt('Updates publication information for publications with a supported database cross-reference.'),
     'options' => array(
-      'create_contacts' => dt('provide this option to create or update contacts for authors. By default contacts are not created or updated.'),
       'dbxref' => dt('An accession number for a publication from a remote database (e.g. PMID:23582642)'),
       'db' => dt('The database name (e.g. PMID or AGL)'),
+      'publish' => dt("Set to 'Y' to publish publications after import into Chado. (default: Y)."),
+      'sync' => dt("For Tripal v3 legacy mode only. Set to 'Y' to sync publications after import into Chado. This will create legacy \"nodes\" rather than the newer Tripal v3 entities."),
+      'username' => array(
+        'description' => dt('The Drupal user name for which the job should be run.  The permissions for this user will be used.'),
+      ),
     ),
     'examples' => array(
-      'Standard example' => 'drush tripal-pubs-update',
-      'Create contacts during update' => 'drush tripal-pubs-update --create_contacts=1',
-      'Update a single record' => 'drush tripal-pubs-update --dbxref=PMID:23582642',
-      'Update all records for a single database' => 'drush tripal-pubs-update --db=PMID'
+      'Standard example' => 'drush trp-update-pubs --username=[username]',
+      'Update a single record' => 'drush trp-update-pubs --username=[username] --dbxref=PMID:23582642',
+      'Update all records for a single database' => 'drush trp-update-pubs --username=[username] --db=PMID'
     ),
   );
 
@@ -126,9 +132,14 @@ function tripal_chado_drush_command() {
  * @ingroup tripal_drush
  */
 function drush_tripal_chado_trp_import_pubs() {
+  // The create contacts only goes with the dbxref argument
   $create_contacts = drush_get_option('create_contacts');
   $dbxref = drush_get_option('dbxref');
+  
+  // The do_report, publish, sync only go with the normal importer.
   $do_report = drush_get_option('report');
+  $publish = drush_get_option('publish');
+  $sync = drush_get_option('sync');
   $update = drush_get_option('update');
   $uname = drush_get_option('username');
 
@@ -140,13 +151,27 @@ function drush_tripal_chado_trp_import_pubs() {
   else {
     $update = FALSE;
   }
+  
+  
+  if ($publish == 'Y' and $sync == 'Y' ) {
+    $publish = 'both';
+  }
+  elseif ($publish != 'Y' and $sync == 'Y') {
+    $publish = 'sync';
+  }
+  elseif ($publish == 'Y' and $sync != 'Y') {
+    $publish = TRUE;
+  }
+  else {
+    $publish = FALSE;
+  }
 
   module_load_include('inc', 'tripal_chado', 'includes/loaders/tripal_chado.pub_importers');
   if ($dbxref) {
-    chado_import_pub_by_dbxref($dbxref, $create_contacts, $update);
+    chado_import_pub_by_dbxref($dbxref, $create_contacts, $publish, $update);
   }
   else {
-    chado_execute_active_pub_importers($do_report, $update);
+    chado_execute_active_pub_importers($do_report, $publish, $update);
   }
 }
 
@@ -159,9 +184,27 @@ function drush_tripal_chado_trp_update_pubs() {
   $create_contacts = drush_get_option('create_contacts');
   $dbxref = drush_get_option('dbxref');
   $db = drush_get_option('db');
+  $publish = drush_get_option('publish');
+  $sync = drush_get_option('sync');
+  
+  $uname = drush_get_option('username');
+  drush_tripal_set_user($uname);
+  
+  if ($publish == 'Y' and $sync == 'Y' ) {
+    $publish = 'both';
+  }
+  elseif ($publish  != 'Y' and $sync == 'Y' ) {
+    $publish = 'sync';
+  }
+  elseif ($publish  == 'Y'  and $sync  != 'Y' ) {
+    $publish = TRUE;
+  }
+  else {
+    $publish = FALSE;
+  }
 
   module_load_include('inc', 'tripal_chado', 'includes/loaders/tripal_chado.pub_importers');
-  chado_reimport_publications($create_contacts, $dbxref, $db);
+  chado_reimport_publications($create_contacts, $dbxref, $db, $publish);
 }
 
 /**

tripal_chado/tripal_chado.module

@@ -1190,6 +1190,66 @@ function tripal_feature_match_features_page($id) {
     ));
     return $output;
   }
+}
 
-
+/**
+ * Implements hook_mail().
+ *
+ * @ingroup tripal_legacy_pub
+ */
+function tripal_chado_mail($key, &$message, $params) {
+  $site_name = variable_get('site_name');
+  $language = $message['language'];
+  switch($key) {
+    case 'import_report':
+      
+      $content = [];
+      $content[] = [
+        '#type' => 'markup',
+        '#markup' => '<h3>Tripal Bulk Publication Report</h3>'
+      ];
+      $report = $params['report'];
+
+      foreach ($report as $action => $pubs) {
+        if (count($pubs) > 0) {
+          $title = count($pubs) . ' publication(s) were ' . $action . ":";
+          if ($action == 'error') {
+            $title = count($pubs) . ' publications were not imported due to errors:';
+          }
+          $content[] = [
+            '#type' => 'item',
+            '#title' => $title, 
+            '#markup' => theme_item_list([
+              'title' => '',
+              'type' => 'ol', 
+              'items' => $pubs,
+              'attributes' => [],
+            ]),
+          ];
+        }
+      }
+      $content = '<html>' . drupal_render($content) . '</html';
+      
+      $message['subject'] = t('Publication import from !site', array('!site' => $site_name));
+      
+      
+      if (module_exists('htmlmail') or module_exists('mimemail')) {
+        $headers = array(
+          'MIME-Version' => '1.0',
+          'Content-Type' => 'text/html; charset=UTF-8; format=flowed',
+          'Content-Transfer-Encoding' => '8Bit',
+          'X-Mailer' => 'Drupal'
+        );
+        foreach ($headers as $key => $value) {
+          $message['headers'][$key] = $value;
+        }
+        $message['body'][] = $content;
+      }
+      else {
+        $message['body'][] =  drupal_html_to_text($content);
+      }
+            
+      break;
+  }
 }
+