소스 검색

Added API functions for inserting Bulk loaders. Also fixed bulk loader so it imports and exports JSON format strings. It also is backwards compatible (needs to be removed eventually) for loading serialized PHP and stringified PHP arrays. The Tripal Extensions Import admin page only supports import of JSON strings for security purposes

Stephen Ficklin 10 년 전
부모
커밋
80b9efb84e

+ 159 - 0
tripal_bulk_loader/api/tripal_bulk_loader.api.templates.inc

@@ -14,6 +14,165 @@
  * @}
  */
 
+/**
+ * Validates an $options array for insert or update of a bulk loader record.
+ *
+ * @param $val_type
+ *   The type of validation. Can be either 'insert' or 'update'.
+ * @param $options
+ *   An array of key/value pairs containing the following keys:
+ *     'template_name':   The name of the template.
+ *     'template_array':  The JSON array representing the template.
+ *   Optional:
+ *     'strict':          If set then only JSON formatted templates are allowed.
+ * @param $errors
+ *   An empty array where validation error messages will be set. The keys
+ *   of the array will be name of the field from the options array and the
+ *   value is the error message.
+ * @param $warnings
+ *   An empty array where validation warning messagges will be set. The
+ *   warnings should not stop an insert or an update but should be provided
+ *   to the user as information by a drupal_set_message() if appropriate. The
+ *   keys of the array will be name of the field from the options array and the
+ *   value is the error message.
+ *
+ * @return
+ *   If validation failes then FALSE is returned.  Any options that do not pass
+ *   validation checks will be added in the $errors array with the key being
+ *   the option and the value being the error message.  If validation
+ *   is successful then TRUE is returned.
+ *
+ */
+function tripal_validate_bulk_loader_template($val_type, &$options, &$errors, &$warnings = array()) {
+  $template_array = trim($options['template_array']);
+  $template_name = trim($options['template_name']);
+  $strict = array_key_exists('strict', $options)  ? $options['strict'] : FALSE;
+
+  // Make sure the template array is one of the supported types
+  // DEPRECATED: A stringified version of the array (causes security issues)
+  if (preg_match('/^array/', $template_array)) {
+    if ($strict) {
+      $errors['template_array'] = t('Invalid template array. Please provide
+        a JSON formatted array');
+      return FALSE;
+    }
+    else {
+      $warnings['template_array'] = t('Please note that import of
+        bulk loader templates as PHP arrays as a stringified array is deprecated
+        and will be removed in future versions of Tripal. Export and import
+        format will be JSON.');
+    }
+  }
+  // DEPRECATED: A serialized PHP array
+  elseif (preg_match('/^a:/', $template_array)) {
+    if ($strict) {
+      $errors['template_array'] = t('Invalid template array. Please provide
+        a JSON formatted array');
+      return FALSE;
+    }
+    else {
+      $warnings['template_array'] = t('Please note that import of
+        bulk loader templates as PHP serialized arrays is deprecated and will
+        be removed in future versions of Tripal. Export and import format will
+        be JSON.');
+    }
+  }
+  // JSON FORMAT
+  elseif (json_decode($template_array)) {
+    // This is correct!
+  }
+  else {
+    $errors['template_array'] = t('The template array must be in
+      JSON format (although PHP strigified arrays and PHP serialized
+      arrays are temporarily supported for backwards compatibility).');
+    return FALSE;
+  }
+
+  // Make sure the template name is unique
+  $name_exists = db_select('tripal_bulk_loader_template', 'tblt')
+    ->fields('tblt', array('template_id'))
+    ->condition('name', $template_name)
+    ->execute()
+    ->fetchField();
+  if ($name_exists) {
+    $errors['template_name'] = t('The template name already exists. Please
+      choose another name.');
+    return FALSE;
+  }
+  return TRUE;
+}
+
+/**
+ * Inserts a bulk loader template record.
+ *
+ * This function validates the options passed prior to insertion of the record,
+ *
+ * @param $options
+ *   An array of key/value pairs containing the following keys:
+ *     'template_name':   The name of the template.
+ *     'template_array':  The JSON array representing the template.
+ *   Optional:
+ *     'strict':          If set then only JSON formatted templates are allowed.
+ * @param $errors
+ *   An empty array where validation error messages will be set. The keys
+ *   of the array will be name of the field from the options array and the
+ *   value is the error message.
+ * @param $warnings
+ *   An empty array where validation warning messagges will be set. The
+ *   warnings should not stop an insert or an update but should be provided
+ *   to the user as information by a drupal_set_message() if appropriate. The
+ *   keys of the array will be name of the field from the options array and the
+ *   value is the error message.
+ * @return
+ *   TRUE for success and FALSE for failure.
+ */
+function tripal_insert_bulk_loader_template($options, &$errors, &$warnings) {
+
+  $success = tripal_validate_bulk_loader_template('insert', $options, $errors, $warnings);
+  if (!$success) {
+    foreach ($errors as $field => $message) {
+      tripal_report_error('tripal_bulkldr', TRIPAL_ERROR, $message);
+    }
+    return FALSE;
+  }
+
+  // Insert the bulk loader template.
+  $template_array = trim($options['template_array']);
+  $template_name = trim($options['template_name']);
+
+  // Previous version of Tripal would export the template as a PHP array.
+  // This has security implications and is deprecated. This support should
+  // be reomved in future versions of Tripal, but to support transfers of
+  // templates between v1.1 and v2.x sites we support it.
+  if (preg_match('/^array/', $template_array)) {
+    $tarray = array();
+    eval("\$tarray = $template_array;");
+    $template_array = serialize($tarray);
+  }
+  // For a brief period, the bulk loader templates were exported as a PHP
+  // serialized array. We have moved to exporting in JSON as JSON is more
+  // user friendly. But we must support the serialized PHP array for
+  // backwards compatibility of v2.0-rc1 sites and v2.x sites.
+  elseif (preg_match('/^a:/', $template_array)) {
+    // do nothing it's in PHP serialized format
+  }
+  // The typical format is JSON
+  elseif (preg_match('/^\{/', $template_array)) {
+    $template_array = serialize(json_decode($template_array, TRUE));
+  }
+
+  $record = array(
+    'name' => $template_name,
+    'template_array' => $template_array,
+    'created' => time(),
+    'changed' => time()
+  );
+  if (!drupal_write_record('tripal_bulk_loader_template', $record)) {
+    return FALSE;
+  }
+  return TRUE;
+}
+
 /**
  * Meant to be called from a form_validate function to ensure a newly added bulk loader record
  * name is unique and not empty.

+ 46 - 16
tripal_bulk_loader/includes/tripal_bulk_loader.admin.templates.inc

@@ -692,7 +692,7 @@ function tripal_bulk_loader_import_template_form($form, &$form_state) {
   $breadcrumb[] = l('Templates', 'admin/tripal/loaders/bulk/templates');
   drupal_set_breadcrumb($breadcrumb);
 
-  $form['new_template_name'] = array(
+  $form['template_name'] = array(
     '#type' => 'textfield',
     '#title' => 'Template Name',
     '#weight' => 1,
@@ -716,6 +716,35 @@ function tripal_bulk_loader_import_template_form($form, &$form_state) {
   return $form;
 }
 
+/**
+ * Validates the import template form
+ */
+function tripal_bulk_loader_import_template_form_validate($form, &$form_state) {
+
+  $options = array(
+    'template_name' => trim($form_state['values']['template_name']),
+    'template_array' => trim($form_state['values']['template_array'])
+  );
+  $errors = array();
+  $warnings = array();
+
+  // Use the API function to Validate the submission.
+  tripal_validate_bulk_loader_template('insert', $options, $errors, $warnings);
+
+  // Now set form errors if any errors were detected.
+  if (count($errors) > 0) {
+    foreach($errors as $field => $message) {
+      form_set_error($field, $message);
+    }
+  }
+  // Add any warnings if any were detected
+  if (count($warnings) > 0) {
+    foreach($warnings as $field => $message) {
+      drupal_set_message($message, 'warning');
+    }
+  }
+}
+
 /**
  * Import Template Form Submit
  *
@@ -727,21 +756,22 @@ function tripal_bulk_loader_import_template_form($form, &$form_state) {
  * @ingroup tripal_bulk_loader
  */
 function tripal_bulk_loader_import_template_form_submit($form, &$form_state) {
-
-  $record = array(
-    'name' => $form_state['values']['new_template_name'],
-    'template_array' => $form_state['values']['template_array'],
-    'created' => time(),
-    'changed' => time()
+  $options = array(
+    'template_name' => trim($form_state['values']['template_name']),
+    'template_array' => trim($form_state['values']['template_array'])
   );
-  $result = drupal_write_record('tripal_bulk_loader_template', $record);
-  if ($result) {
-    drupal_set_message(t('Successfully imported Tripal Bulk Loader Template.'));
-  }
-
-  $form_state['rebuild'] = FALSE;
-  $form_state['redirect'] = 'admin/tripal/loaders/bulk/templates';
+  $errors = array();
+  $warnings = array();
 
+  // Use the API function to insert a bulk loader.
+  if (tripal_insert_bulk_loader_template($options, $errors, $warnings)) {
+    drupal_set_message(t('Successfully imported Tripal bulk loader template.'));
+    $form_state['rebuild'] = FALSE;
+    $form_state['redirect'] = 'admin/tripal/loaders/bulk/templates';
+  }
+  else {
+    drupal_set_message(t('Unable to import Tripal bulk loader template.'), 'error');
+  }
 }
 
 /**
@@ -792,8 +822,8 @@ function tripal_bulk_loader_export_template_form($form, &$form_state) {
   $form['template_array'] = array(
     '#type' => 'textarea',
     '#title' => 'Export',
-    '#default_value' => $form_state['storage']['template_array'],
-    '#description' => t('Use this serialized array for import.'),
+    '#default_value' => json_encode(unserialize($form_state['storage']['template_array'])),
+    '#description' => t('Use this JSON array for import of this bulk loader template into another Tripal site.'),
     '#rows' => 30,
     '#weight' => 5,
 

+ 19 - 13
tripal_core/includes/tripal_core.extensions.inc

@@ -319,6 +319,8 @@ function tripal_core_extension_form_add_extensions(&$form, $form_state, $extensi
         continue;
       }
 
+      $extension['title'] = trim($extension['title']);
+
       // If this is an extension module then there will be a home page for it
       $home_page = '';
       if (array_key_exists('home_page', $extension[$namespace])) {
@@ -347,7 +349,7 @@ function tripal_core_extension_form_add_extensions(&$form, $form_state, $extensi
             ->execute()
             ->fetchField();
           if ($blk_id) {
-            $is_installed = '<li>A bulk loader tempalte with this name is already installed.</li>';
+            $is_installed = '<li>A bulk loader template with this name is already installed.</li>';
           }
           break;
         case 'Materialized View':
@@ -524,20 +526,24 @@ function tripal_core_extensions_form_submit($form, &$form_state) {
     $type = $extension['category'];
     switch ($type) {
       case 'Bulk Loader Template':
-        // TODO: we need an API function for adding a bulk loader
-        $id = db_insert('tripal_bulk_loader_template')
-          ->fields(array(
-            'name' => $extension['title'],
-            'template_array' => $extension[$namespace]['bulkldr_export'],
-            'created' => time(),
-            'changed' => time()
-          ))
-          ->execute();
-        if (!$id) {
-          drupal_set_message("Cannot import this bulk loader. Please view the 'Recent log messages' report for more details.",  'error');
+        $options = array(
+          'template_name' => $extension['title'],
+          'template_array' => $extension[$namespace]['bulkldr_export'],
+          'strict' => TRUE,
+        );
+        $errors = array();
+        $warnings = array();
+        $success = tripal_insert_bulk_loader_template($options, $errors, $warnings);
+        if ($success) {
+          drupal_set_message("Bulk loader succesfully added.");
         }
         else {
-          drupal_set_message("Bulk loader succesfully added.");
+          drupal_set_message("Error importing this bulk loader.",  'error');
+          if (count($errors) > 0) {
+            foreach($errors as $field => $message) {
+              drupal_set_message($message, 'error');
+            }
+          }
         }
         break;
       case 'Materialized View':

+ 34 - 23
tripal_feature/includes/tripal_feature.gff_loader.inc

@@ -389,6 +389,10 @@ function tripal_feature_load_gff3($gff_file, $organism_id, $analysis_id,
   $ret = array();
   $date = getdate();
 
+  // An array that stores CVterms that have been looked up so we don't have
+  // to do the database query every time.
+  $cvterm_lookup = array();
+
   // empty the temp tables
   $sql = "DELETE FROM {tripal_gff_temp}";
   chado_query($sql);
@@ -397,7 +401,6 @@ function tripal_feature_load_gff3($gff_file, $organism_id, $analysis_id,
   $sql = "DELETE FROM {tripal_gffprotein_temp}";
   chado_query($sql);
 
-
   // begin the transaction
   $transaction = null;
   if ($use_transaction) {
@@ -441,7 +444,6 @@ function tripal_feature_load_gff3($gff_file, $organism_id, $analysis_id,
         "Cannot find the 'sequence' ontology", array());
       return '';
     }
-
     // get the organism for which this GFF3 file belongs
     $sql = "SELECT * FROM {organism} WHERE organism_id = :organism_id";
     $organism = chado_query($sql, array(':organism_id' => $organism_id))->fetchObject();
@@ -466,6 +468,23 @@ function tripal_feature_load_gff3($gff_file, $organism_id, $analysis_id,
        (lower(CVT.name) = lower(:name) or lower(CVTS.synonym) = lower(:synonym))
      ";
 
+    // If a landmark type was provided then pre-retrieve that.
+    if ($landmark_type) {
+      $query = array(
+        ':cv_id' => $cv->cv_id,
+        ':name' => $landmark_type,
+        ':synonym' => $landmark_type
+      );
+      $result = chado_query($sel_cvterm_sql, $query);
+      $landmark_cvterm = $result->fetchObject();
+      if (!$landmark_cvterm) {
+        tripal_report_error('tripal_feature', TRIPAL_ERROR,
+          'cannot find landmark feature type \'%landmark_type\'.',
+          array('%landmark_type' => $landmark_type));
+        return '';
+      }
+    }
+
     // iterate through each line of the GFF file
     print "Parsing Line $line_num (0.00%). Memory: " . number_format(memory_get_usage()) . " bytes\r";
     while ($line = fgets($fh)) {
@@ -503,20 +522,7 @@ function tripal_feature_load_gff3($gff_file, $organism_id, $analysis_id,
         $rstart = $region_matches[2];
         $rend = $region_matches[3];
         if ($landmark_type) {
-          $query = array(
-            ':cv_id' => $cv->cv_id,
-            ':name' => $landmark_type,
-            ':synonym' => $landmark_type
-          );
-          $result = chado_query($sel_cvterm_sql, $query);
-          $cvterm = $result->fetchObject();
-          if (!$cvterm) {
-            tripal_report_error('tripal_feature', TRIPAL_ERROR,
-             'cannot find feature type \'%landmark_type\' on line %line_num of the GFF file',
-              array('%landmark_type' => $landmark_type, '%line_num' => $line_num));
-            return '';
-          }
-          tripal_feature_load_gff3_feature($organism, $analysis_id, $cvterm, $rid,
+          tripal_feature_load_gff3_feature($organism, $analysis_id, $landmark_cvterm, $rid,
             $rid, '', 'f', 'f', 1, 0);
         }
         continue;
@@ -573,13 +579,18 @@ function tripal_feature_load_gff3($gff_file, $organism_id, $analysis_id,
       if (strcmp($phase, '.') == 0) {
         $phase = '';
       }
-      $result = chado_query($sel_cvterm_sql, array(':cv_id' => $cv->cv_id, ':name' => $type, ':synonym' => $type));
-
-      $cvterm = $result->fetchObject();
-      if (!$cvterm) {
-        tripal_report_error('tripal_feature', TRIPAL_ERROR, 'cannot find feature term \'%type\' on line %line_num of the GFF file',
-          array('%type' => $type, '%line_num' => $line_num));
-        return '';
+      if (array_key_exists($type, $cvterm_lookup)) {
+        $cvterm = $cvterm_lookup[$type];
+      }
+      else {
+        $result = chado_query($sel_cvterm_sql, array(':cv_id' => $cv->cv_id, ':name' => $type, ':synonym' => $type));
+        $cvterm = $result->fetchObject();
+        $cvterm_lookup[$type] = $cvterm;
+        if (!$cvterm) {
+          tripal_report_error('tripal_feature', TRIPAL_ERROR, 'cannot find feature term \'%type\' on line %line_num of the GFF file',
+            array('%type' => $type, '%line_num' => $line_num));
+          return '';
+        }
       }
 
       // break apart each of the attributes