Added checkbox allowing user to enable/disable keeping track of inserted record IDs on a per bulk loading job basis. The default for already existing bulk loader jobs is TRUE and the default for new bulk loader jobs is FALSE

tripal_bulk_loader/tripal_bulk_loader.admin.inc

@@ -167,6 +167,20 @@ function tripal_bulk_loader_admin_jobs() {
 function tripal_bulk_loader_configuration_form($form_state = NULL) {
   $form = array();
 
+  $form['space'] = array(
+    '#type' => 'fieldset',
+    '#title' => t('Enable/Disable Functionality'),
+  );
+
+  $form['space']['keep_track_inserted'] = array(
+    '#type' => 'checkbox',
+    '#title' => t('Keep track of inserted record IDs'),
+    '#description' => t('This enables the ability to revert an entire loading job even if '
+      .'it completed successfully. Furthermore, it displays the number of records '
+      .'successfully inserted into each table.'),
+    '#default_value' => variable_get('tripal_bulk_loader_keep_track_inserted', FALSE),
+  );
+
   $form['speed'] = array(
     '#type' => 'fieldset',
     '#title' => t('Possible Speed Improvements'),
@@ -249,4 +263,6 @@ function tripal_bulk_loader_configuration_form_submit($form, $form_state) {
   variable_set('tripal_bulk_loader_transactions', $form_state['values']['transactions']);
   variable_set('tripal_bulk_loader_lock', $form_state['values']['lock']);
 
+  variable_set('tripal_bulk_loader_keep_track_inserted', $form_state['values']['keep_track_inserted']);
+
 }

tripal_bulk_loader/tripal_bulk_loader.install

@@ -30,17 +30,17 @@ function tripal_bulk_loader_uninstall() {
 function tripal_bulk_loader_schema() {
   $schema = array();
   $schema['tripal_bulk_loader'] = array(
-      'fields' => array(
-         'nid' => array(
-            'type' => 'int',
-            'unsigned' => TRUE,
-            'not null' => TRUE,
+    'fields' => array(
+      'nid' => array(
+        'type' => 'int',
+        'unsigned' => TRUE,
+        'not null' => TRUE,
       ),
-         'loader_name' => array(
-            'type' => 'varchar',
+      'loader_name' => array(
+        'type' => 'varchar',
       ),
-         'template_id' => array(
-            'type' => 'varchar',
+      'template_id' => array(
+        'type' => 'varchar',
       ),
       'file' => array(
         'type' => 'varchar',
@@ -58,6 +58,12 @@ function tripal_bulk_loader_schema() {
         'not null' => TRUE,
         'default' => 0,
       ),
+      'keep_track_inserted' => array(
+        'type' => 'int',
+        'size' => 'tiny',
+        'not null' => TRUE,
+        'default' => 1
+      ),
     ),
       'primary key' => array('nid'),
       'unique keys' => array(
@@ -190,3 +196,21 @@ function tripal_bulk_loader_update_6151() {
 
   return $ret;
 }
+
+function tripal_bulk_loader_update_6152() {
+  $ret = array();
+
+  db_add_field(
+    $ret,
+    'tripal_bulk_loader',
+    'keep_track_inserted',
+    array(
+      'type' => 'int',
+      'size' => 'tiny',
+      'not null' => TRUE,
+      'default' => 1
+    )
+  );
+
+  return $ret;
+}

tripal_bulk_loader/tripal_bulk_loader.loader.inc

@@ -15,7 +15,8 @@ function tripal_bulk_loader_add_loader_job_form($form_state, $node) {
   $form = array();
 
   // --notify--
-  if ($node->job_status == 'Loading...' AND (variable_get('tripal_bulk_loader_transactions',   'row') != 'none')) {
+  if ($node->job_status == 'Loading...') {
+    $progress = tripal_bulk_loader_progess_file_get_progress($node->job_id);
     drupal_set_message(t("The Loading Summary only updates at the end of each constant set.
       %num records have already been inserted; however, they won't be available until the
       current constant set is full loaded and no errors are encountered.", array('%num' => $progress->num_records)), 'warning');
@@ -46,10 +47,12 @@ function tripal_bulk_loader_add_loader_job_form($form_state, $node) {
     '#value' => 'Cancel Job',
   );
 
-  $form['submit-revert'] = array(
-    '#type' => ($node->job_id) ? 'submit' : 'hidden',
-    '#value' => 'Revert',
-  );
+  if ($node->keep_track_inserted) {
+    $form['submit-revert'] = array(
+      '#type' => ($node->job_id) ? 'submit' : 'hidden',
+      '#value' => 'Revert',
+    );
+  }
 
   return $form;
 }
@@ -330,7 +333,16 @@ function tripal_bulk_loader_load_data($nid, $job_id) {
 
       $data_keys = array_keys($data);
       foreach ($data_keys as $priority) {
-        $status = process_data_array_for_line($priority, $data, $default_data, $field2column, $record2priority, $line, $nid, $num_lines, $group_index);
+        $options = array(
+          'field2column' => $field2column,
+          'record2priority' => $record2priority,
+          'line' => $line,
+          'line_num' => $num_lines,
+          'group_index' => $group_index,
+          'node' => $node,
+          'nid' => $node->nid,
+        );
+        $status = process_data_array_for_line($priority, $data, $default_data, $options);
         tripal_bulk_loader_progress_file_track_job($job_id, $status);
         if (!$status ) {
           // Encountered an error
@@ -378,14 +390,6 @@ function tripal_bulk_loader_load_data($nid, $job_id) {
     tripal_bulk_loader_progress_file_track_job($job_id, FALSE, FALSE, TRUE);
   } //end of foreach constant set
 
-  // check that data was inserted and update job_status
-  $sql = 'SELECT count(*) as num_tables FROM {tripal_bulk_loader_inserted} WHERE nid=%d GROUP BY nid';
-  $result = db_fetch_object(db_query($sql, $nid));
-  if ($result->num_tables > 0) {
-    $node->job_status = 'Data Inserted';
-    drupal_write_record('node', $node, 'nid');
-  }
-
   // set the status of the job (in the node not the tripal jobs)
   if ($loaded_without_errors) {
     $status = 'Loading Completed Successfully';
@@ -400,10 +404,19 @@ function tripal_bulk_loader_load_data($nid, $job_id) {
 /**
  *
  *
+         $options = array(
+          'field2column' => $field2column,
+          'record2priority' => $record2priority,
+          'line' => $line,
+          'line_num' => $num_lines,
+          'group_index' => $group_index,
+          'node' => $node,
+          'nid' => $node->nid,
+        );
  */
-function process_data_array_for_line($priority, &$data, &$default_data, $field2column, $record2priority, $line, $nid, $line_num, $group_index) {
+function process_data_array_for_line($priority, &$data, &$default_data, $addt) {
   $table_data = $data[$priority];
-
+  $addt = (object) $addt;
   $no_errors = TRUE;
 
   $table = $table_data['table'];
@@ -413,23 +426,23 @@ function process_data_array_for_line($priority, &$data, &$default_data, $field2c
 
   //print 'default values:'.print_r($values,TRUE)."\n";
   if ($table_data['need_further_processing']) {
-    $values = tripal_bulk_loader_add_spreadsheetdata_to_values($values, $line, $field2column[$priority]);
+    $values = tripal_bulk_loader_add_spreadsheetdata_to_values($values, $addt->line, $addt->field2column[$priority]);
     if (!$values) {
-      //watchdog('T_bulk_loader', 'Line ' . $line_num . ' Data File Added:' . print_r($values, TRUE), array(), WATCHDOG_NOTICE);
+      //watchdog('T_bulk_loader', 'Line ' . $addt->line_num . ' Data File Added:' . print_r($values, TRUE), array(), WATCHDOG_NOTICE);
     }
 
-    $values = tripal_bulk_loader_add_foreignkey_to_values($values, $data, $record2priority);
+    $values = tripal_bulk_loader_add_foreignkey_to_values($values, $data, $addt->record2priority);
     if (!$values) {
-      //watchdog('T_bulk_loader', 'Line ' . $line_num . ' FK Added:<pre>' . print_r($values, TRUE) . print_r($data[$priority], TRUE) . '</pre>', array(), WATCHDOG_NOTICE);
+      //watchdog('T_bulk_loader', 'Line ' . $addt->line_num . ' FK Added:<pre>' . print_r($values, TRUE) . print_r($data[$priority], TRUE) . '</pre>', array(), WATCHDOG_NOTICE);
     }
   }
-  $values = tripal_bulk_loader_regex_tranform_values($values, $table_data, $line);
+  $values = tripal_bulk_loader_regex_tranform_values($values, $table_data, $addt->line);
   if (!$values) {
-    //watchdog('T_bulk_loader', 'Line ' . $line_num . ' Regex:<pre>' . print_r($values, TRUE) . print_r($table_data, TRUE) . '</pre>' . '</pre>', array(), WATCHDOG_NOTICE);
+    //watchdog('T_bulk_loader', 'Line ' . $addt->line_num . ' Regex:<pre>' . print_r($values, TRUE) . print_r($table_data, TRUE) . '</pre>' . '</pre>', array(), WATCHDOG_NOTICE);
   }
 
   if (!$values) {
-    $msg = 'Line ' . $line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') Aborted due to error in previous record. Values of current record:' . print_r($table_data['values_array'], TRUE);
+    $msg = 'Line ' . $addt->line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') Aborted due to error in previous record. Values of current record:' . print_r($table_data['values_array'], TRUE);
     watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING);
     print "ERROR: " . $msg . "\n";
     $data[$priority]['error'] = TRUE;
@@ -444,7 +457,7 @@ function process_data_array_for_line($priority, &$data, &$default_data, $field2c
       // a field is considered missing if it cannot be null and there is no default
       // value for it or it is of type 'serial'
       if ($def['not null'] == 1 and !array_key_exists($field, $insert_values) and !isset($def['default']) and strcmp($def['type'], serial)!=0) {
-        $msg = 'Line ' . $line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') Missing Database Required Value: ' . $table . '.' . $field;
+        $msg = 'Line ' . $addt->line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') Missing Database Required Value: ' . $table . '.' . $field;
         watchdog('T_bulk_loader', $msg, array(), WATCHDOG_NOTICE);
         $data[$priority]['error'] = TRUE;
       }
@@ -455,7 +468,7 @@ function process_data_array_for_line($priority, &$data, &$default_data, $field2c
   foreach ($table_data['required'] as $field => $required) {
     if ($required) {
       if (!isset($values[$field])) {
-        $msg = 'Line ' . $line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') Missing Template Required Value: ' . $table . '.' . $field;
+        $msg = 'Line ' . $addt->line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') Missing Template Required Value: ' . $table . '.' . $field;
         watchdog('T_bulk_loader', $msg, array(), WATCHDOG_NOTICE);
         $data[$priority]['error'] = TRUE;
       }
@@ -500,7 +513,7 @@ function process_data_array_for_line($priority, &$data, &$default_data, $field2c
     // Use prepared statement?
     if (variable_get('tripal_bulk_loader_prepare', TRUE)) {
       $options = array('statement_name' => 'record_' . $priority);
-      if ($line_num == 1 && $group_index == 1) {
+      if ($addt->line_num == 1 && $addt->group_index == 1) {
         $options['prepare'] = TRUE;
       }
     }
@@ -516,7 +529,7 @@ function process_data_array_for_line($priority, &$data, &$default_data, $field2c
     $record = tripal_core_chado_insert($table, $values, $options);
 
     if (!$record) {
-      $msg = 'Line ' . $line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') Unable to insert record into ' . $table . ' where values:' . print_r($values, TRUE);
+      $msg = 'Line ' . $addt->line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') Unable to insert record into ' . $table . ' where values:' . print_r($values, TRUE);
       watchdog('T_bulk_loader', $msg, array(), WATCHDOG_ERROR);
       print "ERROR: " . $msg . "\n";
       $data[$priority]['error'] = TRUE;
@@ -533,28 +546,30 @@ function process_data_array_for_line($priority, &$data, &$default_data, $field2c
       }
 
       // add to tripal_bulk_loader_inserted
-      $insert_record = db_fetch_object(db_query(
-        "SELECT * FROM {tripal_bulk_loader_inserted} WHERE table_inserted_into='%s' AND nid=%d",
-        $table,
-        $nid
-      ));
-      if ($insert_record) {
-        $insert_record->ids_inserted .= ',' . $values[ $table_desc['primary key'][0] ];
-        drupal_write_record('tripal_bulk_loader_inserted', $insert_record, 'tripal_bulk_loader_inserted_id');
-        //print 'Update: '.print_r($insert_record,TRUE)."\n";
-        return $no_errors;
-      }
-      else {
-        $insert_record = array(
-          'nid' => $nid,
-          'table_inserted_into' => $table,
-          'table_primary_key' => $table_desc['primary key'][0],
-          'ids_inserted' => $values[ $table_desc['primary key'][0] ],
-        );
-        //print 'New: '.print_r($insert_record,TRUE)."\n";
-        $success = drupal_write_record('tripal_bulk_loader_inserted', $insert_record);
-        return $no_errors;
-      }//end of if insert record
+      if ($addt->node->keep_track_inserted) {
+        $insert_record = db_fetch_object(db_query(
+          "SELECT * FROM {tripal_bulk_loader_inserted} WHERE table_inserted_into='%s' AND nid=%d",
+          $table,
+          $addt->nid
+        ));
+        if ($insert_record) {
+          $insert_record->ids_inserted .= ',' . $values[ $table_desc['primary key'][0] ];
+          drupal_write_record('tripal_bulk_loader_inserted', $insert_record, 'tripal_bulk_loader_inserted_id');
+          //print 'Update: '.print_r($insert_record,TRUE)."\n";
+          return $no_errors;
+        }
+        else {
+          $insert_record = array(
+            'nid' => $addt->nid,
+            'table_inserted_into' => $table,
+            'table_primary_key' => $table_desc['primary key'][0],
+            'ids_inserted' => $values[ $table_desc['primary key'][0] ],
+          );
+          //print 'New: '.print_r($insert_record,TRUE)."\n";
+          $success = drupal_write_record('tripal_bulk_loader_inserted', $insert_record);
+          return $no_errors;
+        }//end of if insert record
+      }// end of if keeping track of records inserted
 
     } //end of if insert was successful
   }
@@ -562,7 +577,7 @@ function process_data_array_for_line($priority, &$data, &$default_data, $field2c
     $exists = tripal_core_chado_select($table, array_keys($table_desc['fields']), $values, array('has_record' => TRUE));
     if (!$exists) {
       // No record on select
-      $msg = 'Line ' . $line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') No Matching record in ' . $table . ' where values:' . print_r($values, TRUE);
+      $msg = 'Line ' . $addt->line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') No Matching record in ' . $table . ' where values:' . print_r($values, TRUE);
       watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING);
       $data[$priority]['error'] = TRUE;
       $no_errors = FALSE;

tripal_bulk_loader/tripal_bulk_loader.module

@@ -314,6 +314,17 @@ function tripal_bulk_loader_form($node, $form_state) {
     '#default_value' => $node->file_has_header,
   );
 
+  $form['loader']['keep_track_inserted'] = array(
+    '#type' => 'radios',
+    '#title' => t('Keep track of inserted record IDs'),
+    '#description' => t('This enables the ability to revert an entire loading job even if '
+      .'it completed successfully. Furthermore, it displays the number of records '
+      .'successfully inserted into each table.'),
+    '#options' => array( 1 => 'Yes', 0 => 'No'),
+    '#weight' => -7,
+    '#default_value' => (isset($node->keep_track_inserted)) ? $node->keep_track_inserted : variable_get('tripal_bulk_loader_keep_track_inserted', FALSE),
+  );
+
   return $form;
 }
 
@@ -396,8 +407,8 @@ function tripal_bulk_loader_load($node) {
 function tripal_bulk_loader_insert($node) {
 
   // Insert into tripal_bulk_loader
-  $sql = "INSERT INTO {tripal_bulk_loader} (nid, loader_name, template_id, file, file_has_header, job_status) VALUES (%d, '%s', %d, '%s', %d, '%s')";
-  db_query($sql, $node->nid, $node->loader_name, $node->template_id, $node->file, $node->has_header, 'Initialized');
+  $sql = "INSERT INTO {tripal_bulk_loader} (nid, loader_name, template_id, file, file_has_header, job_status, keep_track_inserted) VALUES (%d, '%s', %d, '%s', %d, '%s', %d)";
+  db_query($sql, $node->nid, $node->loader_name, $node->template_id, $node->file, $node->has_header, 'Initialized', $node->keep_track_inserted);
 
   // Update title
   $node->title =$node->loader_name;
@@ -424,8 +435,8 @@ function tripal_bulk_loader_delete($node) {
 function tripal_bulk_loader_update($node) {
 
   // Update tripal_bulk_loader
-  $sql = "UPDATE {tripal_bulk_loader} SET nid = %d, loader_name = '%s', template_id = %d, file = '%s', file_has_header = '%s' WHERE nid = %d";
-  db_query($sql, $node->nid, $node->loader_name, $node->template_id, $node->file, $node->has_header, $node->nid);
+  $sql = "UPDATE {tripal_bulk_loader} SET nid = %d, loader_name = '%s', template_id = %d, file = '%s', file_has_header = '%s', keep_track_inserted = %d WHERE nid = %d";
+  db_query($sql, $node->nid, $node->loader_name, $node->template_id, $node->file, $node->has_header, $node->keep_track_inserted, $node->nid);
 
   // Add a job if the user want to load the data
   global $user;