Made 'optional' separate for both selects and inserts. Should be backwards compatible

tripal_bulk_loader/tripal_bulk_loader.admin.templates.inc

@@ -134,21 +134,27 @@ function tripal_bulk_loader_modify_template_base_form($form_state = NULL, $mode)
 
         $form['records']['no_records']['#value'] = FALSE;
 
+        $mode_value = '';
+        if ($table_array['optional']) {
+          $mode_value .= 'optional ';
+        } 
+
         // for backwards compatibility we want to convert insert_unique to be 'insert'
+        // and optional to 'insert'
         if (strcmp($table_array['mode'],'insert_unique')==0) {
-          $mode_value = 'insert or select if duplicate';
+          $mode_value .= 'insert or select if duplicate';
         } 
         elseif (strcmp($table_array['mode'],'optional')==0) {
-          $mode_value = 'optional insert';
+          $mode_value .= 'optional insert';
         } 
         elseif (strcmp($table_array['mode'],'insert_once')==0) {
-          $mode_value = 'insert once';
+          $mode_value .= 'insert once';
         } 
         elseif ($table_array['mode']) {
-          $mode_value = $table_array['mode'];
+          $mode_value .= $table_array['mode'];
         }
         else {
-          $mode_value = 'insert';
+          $mode_value .= 'insert';
         }
         
         // add in the select if duplicate 
@@ -156,6 +162,7 @@ function tripal_bulk_loader_modify_template_base_form($form_state = NULL, $mode)
           $mode_value .= ' or select if duplicate';
         } 
         
+        // add in the disabled
         if ($table_array['disable']) {
           $mode_value .= '. <font color="Red">DISABLED</font>';
         } 
@@ -646,16 +653,23 @@ function tripal_bulk_loader_edit_template_record_form(&$form_state = NULL) {
     $table = $form_state['storage']['template_array'][$form_state['storage']['original_priority']]['table'];
   }
   
-  // get the default of the mode
+  // get the default mode
   $mode = $form_state['storage']['template_array'][$form_state['storage']['original_priority']]['mode'];
   if(!$mode){
      $mode = 'insert';
   }
+  
   // get default for the select if duplicate
   $select_if_duplicate = $form_state['storage']['template_array'][$form_state['storage']['original_priority']]['select_if_duplicate'];
   if(!isset($select_if_duplicate)){
     $select_if_duplicate = 1;
   }
+  
+  // get default for the select if duplicate
+  $optional = $form_state['storage']['template_array'][$form_state['storage']['original_priority']]['optional'];
+  if(!isset($optional)){
+    $optional = 0;
+  }
 
   // get the default for disabling the record
   $disable = $form_state['storage']['template_array'][$form_state['storage']['original_priority']]['disable'];
@@ -666,6 +680,11 @@ function tripal_bulk_loader_edit_template_record_form(&$form_state = NULL) {
      $select_if_duplicate = 1;
   }
   
+  // this is just for backwards compatibility. the insert_unique mode type is no longer available
+  if(strcmp($mode,'optional')==0){
+     $mode = 'insert';
+     $optional = 1;
+  }
 
   //dpm($form_state, 'form state');
 
@@ -722,12 +741,18 @@ function tripal_bulk_loader_edit_template_record_form(&$form_state = NULL) {
     '#options' => array(
       'select' => 'SELECT: Don\'t insert this record: it\'s used to define a foreign key in another record',
       'insert' => 'INSERT: Insert the record',
-      'optional' => 'INSERT OPTIONAL: Record will only be inserted if all required data is filled in',
       'insert_once' => 'INSERT ONCE: Record will be inserted once for the entire file',
     ),
     '#default_value' => $mode
   );
   
+  $form['edit_record']['optional'] = array(
+     '#type' => 'checkbox',
+     '#title' => t('Optional'),
+     '#description' => t('The insert or select will only be performed if all required data are present'),
+     '#default_value' => $optional
+  );
+  
   $form['edit_record']['select_if_duplicate'] = array(
      '#type' => 'checkbox',
      '#title' => t('Do INSERT, or SELECT if Duplicate'),
@@ -779,6 +804,7 @@ function tripal_bulk_loader_edit_template_record_form_submit($form, &$form_state
       $record['table'] = $form_state['values']['chado_table'];
       $record['select_if_duplicate'] = $form_state['values']['select_if_duplicate'];
       $record['disable'] = $form_state['values']['disable'];
+      $record['optional'] = $form_state['values']['optional'];
 
       if ($form_state['storage']['original_priority'] != $form_state['values']['field_group']) {
         $record['fields'] = array_merge($record['fields'], $template[ $form_state['values']['field_group'] ]['fields']);

tripal_bulk_loader/tripal_bulk_loader.loader.inc

@@ -165,6 +165,7 @@ function tripal_bulk_loader_load_data($nid, $job_id) {
       $default_data[$priority]['mode'] = ($record_array['mode']) ? $record_array['mode'] : 'insert';
       $default_data[$priority]['select_if_duplicate'] = ($record_array['select_if_duplicate']) ? $record_array['select_if_duplicate'] : 1;
       $default_data[$priority]['disabled'] = ($record_array['disable']) ? $record_array['disable'] : 0;
+      $default_data[$priority]['optional'] = ($record_array['optional']) ? $record_array['optional'] : 0;
       $default_data[$priority]['record_id'] = $record_array['record_id'];
       $record2priority[$record_array['record_id']] = $priority;
       $default_data[$priority]['required'][$field_array['field']] = $field_array['required'];
@@ -267,7 +268,7 @@ function tripal_bulk_loader_load_data($nid, $job_id) {
     $num_records = 0;
     $num_lines = 0;
     $num_errors = 0;
-    $interval = intval($total_lines * 0.10);
+    $interval = intval($total_lines * 0.01);
     if ($interval == 0) {
       $interval = 1;
     }
@@ -327,10 +328,10 @@ function tripal_bulk_loader_load_data($nid, $job_id) {
       $line = explode("\t", $raw_line);
       $num_lines++;
 
-      // update the job status every 10% of lines processed for the current group
+      // update the job status every 1% of lines processed for the current group
       if ($node->job_id and $num_lines % $interval == 0) {
         // percentage of lines processed for the current group
-        $group_progress = round(($num_lines/$total_lines)*100);
+        $group_progress = round(($num_lines / $total_lines) * 100);
         tripal_bulk_loader_progress_bar($num_lines, $total_lines);
         // percentage of lines processed for all groups
         // <previous group index> * 100 + <current group progress>
@@ -339,7 +340,7 @@ function tripal_bulk_loader_load_data($nid, $job_id) {
         // For example, if you were in the third group of 3 constant sets
         // and had a group percentage of 50% then the job progress would be
         // (2*100 + 50%) / 3 = 250%/3 = 83%
-        $job_progress = round(((($group_index-1)*100)+$group_progress)/$total_num_groups);
+        $job_progress = round(((($group_index - 1) * 100) + $group_progress) / $total_num_groups);
         tripal_job_set_progress($node->job_id, $job_progress);
       }
 
@@ -462,36 +463,54 @@ function process_data_array_for_line($priority, &$data, &$default_data, $addt) {
     //watchdog('T_bulk_loader', 'Line ' . $addt->line_num . ' Regex:<pre>' . print_r($values, TRUE) . print_r($table_data, TRUE) . '</pre>' . '</pre>', array(), WATCHDOG_NOTICE);
   }
 
-  if (!$values) {
-    $msg = 'Line ' . $addt->line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') Aborted due to error in previous record. Values of current record:' . print_r($table_data['values_array'], TRUE);
-    watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING);
-    print "ERROR: " . $msg . "\n";
-    $data[$priority]['error'] = TRUE;
-    $no_errors = FALSE;
-  }
+// SPF: commented out 9/5/2012... It is possible a record has no values (it's optional). The
+// checks below I think negate the need for this check, so commented out.
+//  if (!$values) {
+//    $msg = 'Line ' . $addt->line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') Aborted due to error in previous record. Values of current record:' . print_r($table_data['values_array'], TRUE);
+//    watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING);
+//    print "ERROR: " . $msg . "\n";
+//    $data[$priority]['error'] = TRUE;
+//    $no_errors = FALSE;
+//  }
 
+  // get the table description
   $table_desc = tripal_core_get_chado_table_schema($table);
-  if (preg_match('/optional/', $table_array['mode'])) {
+  
+  // check that all database required fields are present if this is in an insert
+  // we check for 'optional' in the mode for backwards compatibility. The 'optional' 
+  // mode used to be a type of insert
+  if (preg_match('/insert/', $table_data['mode']) or preg_match('/optional/', $table_data['mode'])) {
     // Check all db required fields are set
     $fields = $table_desc['fields'];
     foreach ($fields as $field => $def) {
       // a field is considered missing if it cannot be null and there is no default
-      // value for it or it is of type 'serial'
-      if ($def['not null'] == 1 and !array_key_exists($field, $insert_values) and !isset($def['default']) and strcmp($def['type'], serial)!=0) {
+      // value for it or it is not of type 'serial'{
+      if ($def['not null'] == 1 and 
+          !array_key_exists($field, $values) and 
+          !isset($def['default']) and 
+          strcmp($def['type'], serial) != 0) {
         $msg = 'Line ' . $addt->line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') Missing Database Required Value: ' . $table . '.' . $field;
         watchdog('T_bulk_loader', $msg, array(), WATCHDOG_NOTICE);
         $data[$priority]['error'] = TRUE;
       }
     }
-  } //end of if optional record
+  } 
 
-  // Check required fields are present
+  // Check required fields are present if a required field is missing and this
+  // is an optional record then just return. otherwise raise an error
   foreach ($table_data['required'] as $field => $required) {
     if ($required) {
-      if (!isset($values[$field])) {
-        $msg = 'Line ' . $addt->line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') Missing Template Required Value: ' . $table . '.' . $field;
-        watchdog('T_bulk_loader', $msg, array(), WATCHDOG_NOTICE);
-        $data[$priority]['error'] = TRUE;
+      // check if the field has no value (or array is empty)
+      if (!isset($values[$field]) or (is_array($values[$field]) and count($values[$field]) == 0)){
+        if ($table_data['optional']) {
+          return $no_errors;
+        }
+        else {
+          $msg = 'Line ' . $addt->line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') Missing Template Required Value: ' . $table . '.' . $field;
+          watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING);
+          $data[$priority]['error'] = TRUE;
+          $no_errors = FALSE;
+        }
       }
     }
   }
@@ -518,11 +537,11 @@ function process_data_array_for_line($priority, &$data, &$default_data, $addt) {
     $header = $values['uniquename'] . ' ' . $table_data['record_id'];
   }
 
-  // if select if duplicate then check to ensure unique constraint is met.  If so,
-  // then return. We don't want to do any inserts.  We check for insert_unique for backwards compatibilty but that
-  // mode no longer exists
-  if (!preg_match('/insert_unique/', $table_data['mode']) or
-      (isset($table_data['select_if_duplicate']) and $table_data['select_if_duplicate'] == 1)) {
+  // if "select if duplicate" is enabled then check to ensure unique constraint is not violoated.  
+  // If it is violoated then simply return, the record already exists in the database.
+  // We check for insert_unique for backwards compatibilty but that mode no longer exists
+  if (preg_match('/insert_unique/', $table_data['mode']) or
+      $table_data['select_if_duplicate'] == 1) {
     $unique = tripal_core_chado_select($table, array_keys($table_desc['fields']), $values, array('has_record' => TRUE));
     //print 'Unique?'.print_r(array('table' => $table, 'columns' => array_keys($table_desc['fields']), 'values' => $values),TRUE).' returns '.$unique."\n";
     if ($unique > 0) {
@@ -599,13 +618,14 @@ function process_data_array_for_line($priority, &$data, &$default_data, $addt) {
   }
   else {
     $exists = tripal_core_chado_select($table, array_keys($table_desc['fields']), $values, array('has_record' => TRUE));
-    if (!$exists) {
+    // if the record doesn't exists and it's not optional then generate an error
+    if (!$exists and $table_data['optional'] != 1) {
       // No record on select
       $msg = 'Line ' . $addt->line_num . ' ' . $table_data['record_id'] . ' (' . $table_data['mode'] . ') No Matching record in ' . $table . ' where values:' . print_r($values, TRUE);
       watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING);
       $data[$priority]['error'] = TRUE;
       $no_errors = FALSE;
-    }
+    } 
   }
   return $no_errors;
 }
@@ -762,7 +782,7 @@ function tripal_bulk_loader_progress_bar($current=0, $total=100, $size=50) {
   }
 
   //Percentage round off for a more clean, consistent look
-  $perc = round(($current/$total)*100, 2);
+  $perc = sprintf("%.02f",round(($current/$total)*100, 2));
   // percent indicator must be four characters, if shorter, add some spaces
   for ($i = strlen($perc); $i <= 4; $i++) {
     $perc = ' ' . $perc;