Tripal: Bulk Loader -support for optional records and required fields (fields not required by chado but that you would like required)

theme_tripal/tripal_bulk_loader/tripal_bulk_loader_modify_template_base_form.tpl.php

@@ -10,7 +10,7 @@
     
     // generate table
     drupal_add_tabledrag('draggable-table', 'order', 'sibling', 'records-reorder');
-    $header = array('Record Name', 'Chado Table', 'Order', '');
+    $header = array('Record Name', 'Chado Table', 'Action', 'Order', '');
     $rows = array();
     foreach (element_children($form['records']['records-data']) as $key) {
       $element = &$form['records']['records-data'][$key];
@@ -19,6 +19,7 @@
       $row = array();
       $row[] = drupal_render($element['title']);
       $row[] = drupal_render($element['chado_table']);
+      $row[] = drupal_render($element['mode']);
       $row[] = drupal_render($element['new_priority']) . drupal_render($element['id']);
       $row[] = drupal_render($element['submit-edit_record']) . '<br>' . drupal_render($element['submit-add_field']);
       $rows[] = array('data' => $row, 'class' => 'draggable');

tripal_bulk_loader/tripal_bulk_loader.admin.inc

@@ -147,6 +147,10 @@ function tripal_bulk_loader_modify_template_base_form ($form_state = NULL, $mode
             '#type' => 'markup',
             '#value' => $table_array['table'],
           ),
+          'mode' => array(
+            '#type' => 'item',
+            '#value' => ($table_array['mode']) ? $table_array['mode'] : 'insert',
+          ),
           'new_priority' => array(
             '#type' => 'select',
             '#options' => range(1, sizeof($form_state['storage']['template'])),
@@ -642,6 +646,7 @@ function tripal_bulk_loader_edit_template_record_form (&$form_state = NULL) {
     '#options' => array(
       'insert' => 'Insert the record if it doesn\'t already exist',
       'select' => 'Don\'t insert this record: it\'s used to define a foreign key in another record',
+      'optional' => 'Record will only be inserted if all required data is filled in'
     ),
     '#default_value' => 'insert'
   );
@@ -1383,7 +1388,17 @@ function tripal_bulk_loader_edit_template_field_form (&$form_state = NULL) {
     ),
   );
 
+  $form['edit_fields']['additional'] = array(
+    '#type' => 'fieldset',
+    '#title' => 'Additional Options',
+  );
 
+  $form['edit_fields']['additional']['required'] = array(
+    '#type' => 'checkbox',
+    '#title' => 'Make this file required',
+    '#default_value' => FALSE,
+  );
+  
   $form['edit_fields']['submit-edit_field'] = array(
       '#type' => 'submit',
       '#value' => 'Edit Field'
@@ -1433,7 +1448,7 @@ function tripal_bulk_loader_edit_template_field_form_submit ($form, &$form_state
           'type' => 'table field',
           'title' => $form_state['values']['field_title'],
           'field' => $form_state['values']['chado_field'],
-          //'required' => <true|false>,
+          'required' => $form_state['values']['required'],
           //'allowed values' => empty by default,
           'spreadsheet sheet' => $form_state['values']['sheet_name'],
           'spreadsheet column' => $form_state['values']['column_number'],
@@ -1448,7 +1463,7 @@ function tripal_bulk_loader_edit_template_field_form_submit ($form, &$form_state
           'type' => 'constant',
           'title' => $form_state['values']['field_title'],
           'field' => $form_state['values']['chado_field'],
-          //'required' => <true|false>,
+          'required' => $form_state['values']['required'],
           //'allowed values' => empty by default,
           'constant value' => $form_state['values']['constant_value'],
           //'exposed' => 'true|false'  If exposed, will give a select box first from allowed values if set, second from database if values not set.     
@@ -1459,10 +1474,12 @@ function tripal_bulk_loader_edit_template_field_form_submit ($form, &$form_state
           'title' => $form_state['values']['field_title'],
           'field' => $form_state['values']['chado_field'],
           'foreign key' => $form_state['values']['foreign_record'],
-          //'required' => <true|false>,
+          'required' => $form_state['values']['required'],
         );      
       }
 
+      // Deal with any additional options
+      dpm($form_state['values'], 'values');
       
       // if the record has changed...
       $form_state['storage']['template_array'][$priority]['table'] = $form_state['values']['chado_table'];

tripal_bulk_loader/tripal_bulk_loader.info

@@ -4,4 +4,4 @@ core = 6.x
 project = tripal_bulk_loader
 package = Tripal
 dependencies[] = tripal_core
-version = "0.4a"
+version = "0.5a"

tripal_bulk_loader/tripal_bulk_loader.loader.inc

@@ -51,13 +51,11 @@ function tripal_bulk_loader_add_loader_job_form_submit ($form, $form_state) {
 			$job_id = tripal_add_job("Bulk Loading Job: $fname",'tripal_bulk_loader', 'tripal_bulk_loader_load_data', $job_args, $user->uid);
 			
 			// add job_id to bulk_loader node
-      $record = node_load($form_state['values']['nid']);
-      $record->job_id = $job_id;
-      drupal_write_record('tripal_bulk_loader', $record, 'nid');
-
+      $success = db_query("UPDATE {tripal_bulk_loader} SET job_id=%d WHERE nid=%d", $job_id, $form_state['values']['nid']);
+      
       // Add Sync Features Job
-      tripal_add_job('Sync all features','tripal_feature',
-        'tripal_feature_sync_features',$job_args,$user->uid);			
+//      tripal_add_job('Sync all features','tripal_feature',
+//        'tripal_feature_sync_features',$job_args,$user->uid);			
 		} else {
 			drupal_set_message("Can not open ".$form_state['values']['file'].". Job not scheduled.");
 		}    
@@ -97,8 +95,10 @@ function tripal_bulk_loader_load_data($nid) {
     foreach ($record_array['fields'] as $field_index => $field_array) {
       $default_data[$priority]['table'] = $record_array['table'];
       $default_data[$priority]['mode'] = ($record_array['mode']) ? $record_array['mode'] : 'insert';
+      $default_data[$priority]['record_id'] = $record_array['record_id'];
       $record2priority[$record_array['record_id']] = $priority;
-      
+      $default_data[$priority]['required'][$field_array['field']] = $field_array['required'];
+        
       if (preg_match('/table field/', $field_array['type'])) {
         $default_data[$priority]['values_array'][$field_array['field']] = '';
         $default_data[$priority]['need_further_processing'] = TRUE;
@@ -131,6 +131,7 @@ function tripal_bulk_loader_load_data($nid) {
     $line = array();
     $raw_line = fgets($file_handle, 4096);
     $raw_line = trim($raw_line);
+    if (preg_match('/^\s*$/', $raw_line)) { continue; } // skips blank lines
     $line = preg_split("/\t/", $raw_line);
     $num_lines++;
 
@@ -144,36 +145,80 @@ function tripal_bulk_loader_load_data($nid) {
         $values = tripal_bulk_loader_add_foreignkey_to_values($values, $data, $record2priority);
       }
       
+      if (!$values) {
+        $msg = $table_data['record_id'].' ('.$table_data['mode'].') Aborted due to error in previous record.';
+        watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING); 
+        print "ERROR: ".$msg."\n";
+        $data[$priority]['error'] = TRUE;
+      }
+      
+      if (preg_match('/optional/', $table_array['mode'])) {
+        // Check all required fields are set
+        $table_desc = module_invoke_all('chado_'.$table.'_schema');
+        $fields = $table_desc['fields'];
+        foreach($fields as $field => $def){
+          // a field is considered missing if it cannot be null and there is no default
+          // value for it or it is of type 'serial'
+          if($def['not null'] == 1 and !array_key_exists($field,$insert_values) and !isset($def['default']) and strcmp($def['type'],serial)!=0){
+             $msg = $table_data['record_id'].' ('.$table_data['mode'].') Missing Database Required Value: '.$table.'.'.$field;
+             watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING); 
+            print "ERROR: ".$msg."\n";
+            $data[$priority]['error'] = TRUE;
+          }
+        }
+      } //end of if optional record
+      
+      // Check required fields are present
+      foreach ($table_data['required'] as $field => $required) {
+        if ($required) {
+          if (!isset($values[$field])) {
+            $msg = $table_data['record_id'].' ('.$table_data['mode'].') Missing Template Required Value: '.$table.'.'.$field;
+            watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING); 
+            print "ERROR: ".$msg."\n";
+            $data[$priority]['error'] = TRUE;            
+          }
+        }
+      }
+      
+      if ($data[$priority]['error']) {
+        continue;
+      }
+      
       // add new values array into the data array
       $data[$priority]['values_array'] = $values;
        
       // first check if it already exists
       $exists = tripal_core_chado_select($table, array_keys($values), $values, array('has_record'=>TRUE));
-      if ($exists) {
+      //print "Number of Records:".$exists.".\n";
+      if ($exists === 1) {
         if (!preg_match('/select/',$table_data['mode'])) {
-          watchdog('T_bulk_loader', 
-            'Record already exists in %table: %record',
-            array('%table' => $table, '%record' => tripal_bulk_loader_flatten_array($values)),
-            'WATCHDOG_WARNING'
-          );  
+          $msg = $table_data['record_id'].' ('.$table_data['mode'].') Record already exists in '.$table.' where values:'.print_r($values,TRUE);
+          watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING); 
+          print "ERROR: ".$msg."\n";
+          $data[$priority]['error'] = TRUE;
         }
       } elseif ($exists > 1) { 
-          watchdog('T_bulk_loader', 
-            'More than 1 record exists in %table: %record',
-            array('%table' => $table, '%record' => tripal_bulk_loader_flatten_array($values)),
-            'WATCHDOG_WARNING'
-          );          
+          $msg = $table_data['record_id'].' ('.$table_data['mode'].') 2+ Records exists in '.$table.' where values:'.print_r($values,TRUE);
+          watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING); 
+          print "ERROR: ".$msg."\n";   
+          $data[$priority]['error'] = TRUE;
       } else {
         // if it doesn't exist already then insert it
-        if (preg_match('/insert/',$table_data['mode'])) {
+        if (!preg_match('/select/',$table_data['mode'])) {
           $success = tripal_core_chado_insert($table, $values);
           if (!$success) {
-            watchdog('T_bulk_loader', 
-              'Unable to insert the following record into %table: %record',
-              array('%table' => $table, '%record' => tripal_bulk_loader_flatten_array($values)),
-              'WATCHDOG_ERROR'
-            );
+            $msg = $table_data['record_id'].' ('.$table_data['mode'].') Unable to insert record into '.$table.' where values:'.print_r($values,TRUE);
+            watchdog('T_bulk_loader', $msg, array(), WATCHDOG_ERROR); 
+            print "ERROR: ".$msg."\n";  
+            $data[$priority]['error'] = TRUE;
+          } else {
+            //print "Inserted ".$table_data['record_id']." into ".$table."\n";
           }
+        } else {
+          $msg = $table_data['record_id'].' ('.$table_data['mode'].') No Matching record in '.$table.' where values:'.print_r($values,TRUE);
+          watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING); 
+          print "ERROR: ".$msg."\n";
+          $data[$priority]['error'] = TRUE;        
         }
       }// end of if/not record exists
     } // end of foreach table in default data array
@@ -200,8 +245,10 @@ function tripal_bulk_loader_add_spreadsheetdata_to_values ($values, $line, $fiel
     if (is_array($value)) { continue; }
     
     $column = $field2column[$field] - 1;
-    if ($line[$column] OR (!$values[$field])) {
+    if (preg_match('/\w+/',$line[$column])) {
       $values[$field] = $line[$column];
+    } else {
+      unset($values[$field]);
     }
   }
   
@@ -225,7 +272,11 @@ function tripal_bulk_loader_add_foreignkey_to_values($values, $data, $record2pri
       $foreign_values = $data[$foreign_priority]['values_array'];
       
       //add to current values array
-      $values[$field] = $foreign_values;
+      if (!$data[$foreign_priority]['error']) {
+        $values[$field] = $foreign_values;
+      } else {
+        return FALSE;
+      }
     }
   }
 

tripal_bulk_loader/tripal_bulk_loader.module

@@ -201,7 +201,7 @@ function tripal_bulk_loader_form ($node){
    	$form['label'] = array(
    	'#type' => 'item',
       '#description' => t("Loader template needs to be created before any bulk loader can be added. Go to 'Tripal Management > Bulk Loader Template' to create the template."),
-      '#weight'        => 0,
+      '#weight'        => -10,
 		);
 		
 		return $form;
@@ -210,7 +210,7 @@ function tripal_bulk_loader_form ($node){
    $form['loader_name'] = array(
    	'#type'          => 'textfield',
       '#title'         => t('Loading Job Name'),
-      '#weight'        => -3,
+      '#weight'        => -10,
       '#required'      => TRUE,
    	'#default_value' => $node->loader_name
    );
@@ -220,7 +220,7 @@ function tripal_bulk_loader_form ($node){
       '#title' => t('Template'),
    	'#description'   => t('Please specify a template for this loader'),
     	'#options'       => $templates,
-   	'#weight'        => -2,
+   	'#weight'        => -9,
       '#required'      => TRUE,
    	'#default_value' => $node->template_id
    );
@@ -229,9 +229,17 @@ function tripal_bulk_loader_form ($node){
       '#type'          => 'textfield',
       '#title'         => t('Data File'),
       '#description'   => t('Please specify the data file to be loaded.'),
-      '#weight'        => -1,
+      '#weight'        => -8,
    	'#default_value' => $node->file
    );
+   
+   $form['has_header'] = array(
+    '#type' => 'radios',
+    '#title' => 'File has a Header',
+    '#options' => array( 't' => 'Yes', 'f' => 'No'),
+    '#weight' => -7,
+    '#default_value' => $node->file_has_header,
+   );
 
    return $form;
 }
@@ -290,24 +298,13 @@ function tripal_bulk_loader_load($node){
  * tripal_bulk_loader_insert
  */
 function tripal_bulk_loader_insert ($node) {
-	$sql = "INSERT INTO {tripal_bulk_loader} (nid, loader_name, template_id, file) VALUES (%d, '%s', %d, '%s')";
-	db_query($sql, $node->nid, $node->loader_name, $node->template_id, $node->file);
+  
+	$sql = "INSERT INTO {tripal_bulk_loader} (nid, loader_name, template_id, file, file_has_header) VALUES (%d, '%s', %d, '%s', %d)";
+	db_query($sql, $node->nid, $node->loader_name, $node->template_id, $node->file, $node->has_header);
 	$node->title =$node->loader_name;
 	drupal_write_record('node',$node,'nid');
 	drupal_write_record('node_revision',$node,'nid');	
-	// Add a job if the user want to load the data
-	global $user;
-	if($node->job) {
-		$job_args[0] =$node->loader_name;
-		$job_args[1] = $node->template_id;
-		$job_args[2] = $node->file;
-		if (is_readable($node->file)) {
-			$fname = preg_replace("/.*\/(.*)/", "$1", $node->file);
-			tripal_add_job("Bulk Load: $fname",'tripal_bulk_loader', 'tripal_bulk_loader_load_data', $job_args, $user->uid);
-		} else {
-			drupal_set_message("Can not open $node->file. Job not scheduled.");
-		}
-	}
+
 }
 
 /**
@@ -322,8 +319,8 @@ function tripal_bulk_loader_delete ($node) {
  * tripal_bulk_loader_update
  */
 function tripal_bulk_loader_update ($node) {
-	$sql = "UPDATE {tripal_bulk_loader} SET nid = %d, loader_name = '%s', template_id = %d, file = '%s' WHERE nid = %d";
-	db_query($sql, $node->nid, $node->loader_name, $node->template_id, $node->file, $node->nid);
+	$sql = "UPDATE {tripal_bulk_loader} SET nid = %d, loader_name = '%s', template_id = %d, file = '%s', file_has_header = '%s' WHERE nid = %d";
+	db_query($sql, $node->nid, $node->loader_name, $node->template_id, $node->file, $node->has_header, $node->nid);
 	// Add a job if the user want to load the data
 	global $user;
 	if($node->job) {

tripal_core/tripal_core.api.inc

@@ -183,7 +183,7 @@ function tripal_core_chado_insert($table,$values){
    foreach($fields as $field => $def){
       // a field is considered missing if it cannot be null and there is no default
       // value for it or it is of type 'serial'
-      if($def['not null'] == 1 and !array_key_exists($field,$insert_values) and !$def['default'] and strcmp($def['type'],serial)!=0){
+      if($def['not null'] == 1 and !array_key_exists($field,$insert_values) and !isset($def['default']) and strcmp($def['type'],serial)!=0){
          watchdog('tripal_core',"tripal_core_chado_insert: Field $table.$field cannot be null: " . print_r($values,1),array(),'WATCHDOG_ERROR');
          return false;
       }