Tripal: Some needed stock changes; bulk loader updated but not working

theme_tripal/tripal_bulk_loader/tripal_bulk_loader_edit_template_base_form.tpl.php

@@ -3,42 +3,69 @@
 
 <?php print drupal_render($form['template_name']); ?>
 
+<!-- For each table display details in a draggable table -->
+<?php if (!$form['records']['no_records']['#value']) { ?>
+  <fieldset><legend><?php print $form['records']['#title']; ?></legend>
+  <?php 
+    print drupal_render($form['records']['description']);
+    
+    // generate table
+    drupal_add_tabledrag('draggable-table', 'order', 'sibling', 'records-reorder');
+    $header = array('Record Name', 'Chado Table', 'Order');
+    $rows = array();
+    foreach (element_children($form['records']['records-data']) as $key) {
+      $element = &$form['records']['records-data'][$key];
+      $element['new_priority']['#attributes']['class'] = 'records-reorder';
+      
+      $row = array();
+      $row[] = drupal_render($element['title']);
+      $row[] = drupal_render($element['chado_table']);
+      $row[] = drupal_render($element['new_priority']) . drupal_render($element['id']);
+      $rows[] = array('data' => $row, 'class' => 'draggable');
+    }
+    
+    print theme('table', $header, $rows, array('id' => 'draggable-table'));
+  
+    // Render submit
+    print drupal_render($form['records']['submit-reorder']);
+    unset($form['records']);
+  ?>
+  </fieldset>
+<?php } ?>
+
 <!-- For each field display details plus edit/delete buttons-->
-<?php if ($form['fields']['total_fields']['#value'] > 0) {?>
-<fieldset><legend>Current Fields</legend>
-  <table>
-    <tr>
-      <th>Field Name</th>
-      <th>Record Group</th>
-      <th>Chado Table</th>
-      <th>Chado Field</th>
-      <th>Worksheet</th>
-      <th>Column</th>
-      <th>Constant Value</th>
-      <th></th>
-    </tr>
-  <?php for($i=1; $i<$form['fields']['total_fields']['#value']; $i++) { ?>
-    <tr>
-      <td><?php print drupal_render($form['fields']["field_name-$i"]);?></td>
-      <td><?php print drupal_render($form['fields']["field_group-$i"]);?></td>
-      <td><?php print drupal_render($form['fields']["chado_table_name-$i"]);?></td>
-      <td><?php print drupal_render($form['fields']["chado_field_name-$i"]);?></td>
-      <td><?php print drupal_render($form['fields']["sheet_name-$i"]);?></td>
-      <td><?php print drupal_render($form['fields']["column_num-$i"]);?></td>
-      <td><?php print drupal_render($form['fields']["constant_value-$i"]);?></td>
-      <td>
-        <?php print drupal_render($form['fields']["edit-$i"]);?>
-        <?php print drupal_render($form['fields']["delete-$i"]);?>
-        <?php print drupal_render($form['fields']["field_index-$i"]);?>
-      </td>
-    </tr>
-  <?php } ?>
-  </table>
-<?php 
-  } 
-  unset($form['fields']);
-?>
-</fieldset>
+<?php if ($form['fields']['total_fields']['#value'] > 0) { ?>
+  <fieldset><legend><?php print $form['fields']['#title']; ?></legend>
+
+  <?php 
+    // generate table
+    $header = array('Record Name', 'Field Name', 'Chado Table', 'Chado Field', 'Worksheet', 'Column', 'Constant Value', 'Foreign Record', '');
+    $rows = array();
+    foreach ($form['fields']['fields-data'] as $key => $element) {
+      if (preg_match('/^#/', $key)) { continue; }
+      
+      $row = array();
+      $row[] = drupal_render($element['record_id']);
+      $row[] = drupal_render($element['field_name']);
+      $row[] = drupal_render($element['chado_table_name']);
+      $row[] = drupal_render($element['chado_field_name']);
+      $row[] = drupal_render($element['sheet_name']);
+      $row[] = drupal_render($element['column_num']);
+      $row[] = drupal_render($element['constant_value']);
+      $row[] = drupal_render($element['foreign_record_id']);
+      $row[] = drupal_render($element['edit_submit']) . '<br>' . drupal_render($element['delete_submit']);
+      
+      $rows[] = $row;
+    }
+    print theme('table', $header, $rows, array());
+    
+    // Render other elements
+    print drupal_render($form['fields']['add_field']);
+    unset($form['fields']);
+  ?>
+  </fieldset>
+<?php } ?>
+
 <!-- Display Rest of form -->
 <?php print drupal_render($form); ?>
 </div>

theme_tripal/tripal_bulk_loader/tripal_bulk_loader_template.tpl.php

@@ -1,44 +1,44 @@
 <?php
-	//dpm($template->template_array, 'Template Array (tpl)');
 	$fields = array();
 	$constants = array();
-	foreach ($template->template_array as $table => $table_groups) {
-		if (!is_array($table_groups)) {
+	foreach ($template->template_array as $priority => $table_array) {
+		if (!is_array($table_array)) {
 			continue;
 		}
 		
-		foreach ($table_groups as $group => $table_array) {
-      foreach ($table_array['field'] as $field) {
-        if (preg_match('/table field/', $field['type'])) {
-          $field['table'] = $table;
-          $field['group'] = $group;
-          $sheet = $field['spreadsheet sheet'];
-          $column = $field['spreadsheet column'];
-          $fields[$sheet.'-'.$column][] = $field;
-        } elseif ($field['type'] == 'constant') {
-          $field['table'] = $table;
-          $field['group'] = $group;
-          $constants[] = $field;
-        }
-      }
-    }
+		$table = $table_array['table'];
+		$record = $table_array['record_id'];
+		foreach ($table_array['fields'] as $field) {
+			if (preg_match('/table field/', $field['type'])) {
+				$field['table'] = $table;
+				$field['record'] = $record;
+				$sheet = $field['spreadsheet sheet'];
+				$column = $field['spreadsheet column'];
+				$fields[$sheet.'-'.$column][] = $field;
+			} elseif ($field['type'] == 'constant') {
+				$field['table'] = $table;
+				$field['record'] = $record;
+				$constants[] = $field;
+			}
+		}
 	}
 ?>
 
 <div id="tripal_bulk_loader-base-box" class="tripal_bulk_loader-info-box tripal-info-box">
   <div class="tripal_bulk_loader-info-box-title tripal-info-box-title">Template Description</div>
   <div class="tripal_bulk_loader-info-box-desc tripal-info-box-desc"></div>
-  
+
+<?php if (sizeof($constants)) { ?>  
   <table id="tripal_bulk_loader-template_constant-table" class="tripal_bulk_loader-table tripal-table tripal-table-vert">
   <caption><b>Constants</b> -These values are applied to all records in the Spreadsheet</caption>
-    <tr><th rowspan="2">Field Name</th><th rowspan="2">Value</th><th colspan="3">Chado Database</th></tr>
-    <tr><th>Group</th><th>Table</th><th>Field</th></tr>
+    <tr><th rowspan="2">Record Name</th><th rowspan="2">Field Name</th><th rowspan="2">Value</th><th colspan="2">Chado Database</th></tr>
+    <tr><th>Table</th><th>Field</th></tr>
     <?php $row = 'even' ?>
     <?php foreach ($constants as $field) {?>
       <tr class="tripal_bulk_loader-table-<?php print $row; ?>-row tripal-table-<?php print $row; ?>-row">
+				<td><?php print $field['record'];?></td>
         <td><?php print $field['title'];?></td>
         <td><?php print $field['constant value']; ?></td>
-        <td><?php print $field['group'];?></td>
         <td><?php print $field['table'];?></td>
         <td><?php print $field['field'];?></td>	
       </tr>
@@ -46,22 +46,24 @@
     <?php } ?>
   </table>
   
+<?php } if (sizeof($fields)) { ?>  
   <table id="tripal_bulk_loader-template_fields-table" class="tripal_bulk_loader-table tripal-table tripal-table-vert">
   <caption><b>Fields</b> -Below is a mapping between Spreadsheet columns and the Chado Database</caption>
-  <tr><th rowspan="2">Field Name</th><th colspan="2">Spreadsheet</th><th colspan="3">Chado Datbase</th></tr>
-  <tr><th>Worksheet</th><th>Column</th><th>Group</th><th>Table</th><th>Field</th></tr>
+  <tr><th rowspan="2">Record Name</th><th rowspan="2">Field Name</th><th colspan="2">Spreadsheet</th><th colspan="2">Chado Datbase</th></tr>
+  <tr><th>Worksheet</th><th>Column</th><th>Table</th><th>Field</th></tr>
   <?php $row = 'even' ?>
   <?php foreach ($fields as $column) {?>
     <?php foreach ($column as $field) {?>
     <tr class="tripal_bulk_loader-table-<?php print $row; ?>-row tripal-table-<?php print $row; ?>-row">
+    	<td><?php print $field['record'];?></td>
       <td><?php print $field['title'];?></td>
       <td><?php print $field['spreadsheet sheet']; ?></td>
       <td><?php print $field['spreadsheet column'];?></td>
-      <td><?php print $field['group'];?></td>
       <td><?php print $field['table'];?></td>
       <td><?php print $field['field'];?></td>
     <tr>
     <?php $row = ($row == 'odd') ? 'even':'odd' ; ?>
   <?php }} ?>
   </table>
+<?php } ?>
 </div>

tripal_bulk_loader/tripal_bulk_loader.info_hooks.inc

@@ -1,7 +1,10 @@
 <?php
 
 /**
- * Implements hook_tripal_bulk_loader_supported_modules
+ * Implements hook_tripal_bulk_loader_supported_modules()
+ * This is used to supply the base tables or modules supported by the bulk loader.
+ * If a base table is included in this list there must also be a corresponding 
+ * implementation of hook_tripal_bulk_loader_<base table>_related_tables()
  * 
  * @return 
  *   An array of <chado table> => <Human-readable Name> describing the modules supported
@@ -12,9 +15,23 @@ function tripal_bulk_loader_tripal_bulk_loader_supported_modules() {
     'organism' => 'Organism',
     'library' => 'Library',
     'analysis' => 'Analysis',
+    'stock' => 'Stocks',
+    'genotype' => 'Genotypes',
   );
 }
 
+/**
+ * Implements hook_tripal_bulk_loader_<base table>_related_tables()
+ * This is used to supply the bulk loader with a list of tables and their 
+ * relationship to the base table.
+ *
+ * @return
+ * An array of relationships (any combination of 'Base Table', 'Foreign Key Relations', 
+ * 'Direct Relations', 'Indirect Relations') where each relationship is keyed by the type
+ * of relationship and the value is an array of tables with that relationship to the base table.
+ * Each table in the array of tables with that relationship should be of the form
+ * <chado table> => <Human-readable Name>.
+ */
 function tripal_bulk_loader_tripal_bulk_loader_feature_related_tables () {
 
   return array(
@@ -50,6 +67,18 @@ function tripal_bulk_loader_tripal_bulk_loader_feature_related_tables () {
 
 }
 
+/**
+ * Implements hook_tripal_bulk_loader_<base table>_related_tables()
+ * This is used to supply the bulk loader with a list of tables and their 
+ * relationship to the base table.
+ *
+ * @return
+ * An array of relationships (any combination of 'Base Table', 'Foreign Key Relations', 
+ * 'Direct Relations', 'Indirect Relations') where each relationship is keyed by the type
+ * of relationship and the value is an array of tables with that relationship to the base table.
+ * Each table in the array of tables with that relationship should be of the form
+ * <chado table> => <Human-readable Name>.
+ */
 function tripal_bulk_loader_tripal_bulk_loader_organism_related_tables () {
   return array(
     'Base Table' => array(
@@ -69,6 +98,18 @@ function tripal_bulk_loader_tripal_bulk_loader_organism_related_tables () {
   );
 }
 
+/**
+ * Implements hook_tripal_bulk_loader_<base table>_related_tables()
+ * This is used to supply the bulk loader with a list of tables and their 
+ * relationship to the base table.
+ *
+ * @return
+ * An array of relationships (any combination of 'Base Table', 'Foreign Key Relations', 
+ * 'Direct Relations', 'Indirect Relations') where each relationship is keyed by the type
+ * of relationship and the value is an array of tables with that relationship to the base table.
+ * Each table in the array of tables with that relationship should be of the form
+ * <chado table> => <Human-readable Name>.
+ */
 function tripal_bulk_loader_tripal_bulk_loader_library_related_tables () {
   return array(
     'Base Table' => array(
@@ -90,6 +131,18 @@ function tripal_bulk_loader_tripal_bulk_loader_library_related_tables () {
   );
 }
 
+/**
+ * Implements hook_tripal_bulk_loader_<base table>_related_tables()
+ * This is used to supply the bulk loader with a list of tables and their 
+ * relationship to the base table.
+ *
+ * @return
+ * An array of relationships (any combination of 'Base Table', 'Foreign Key Relations', 
+ * 'Direct Relations', 'Indirect Relations') where each relationship is keyed by the type
+ * of relationship and the value is an array of tables with that relationship to the base table.
+ * Each table in the array of tables with that relationship should be of the form
+ * <chado table> => <Human-readable Name>.
+ */
 function tripal_bulk_loader_tripal_bulk_loader_analysis_related_tables () {
   return array(
     'Base Table' => array(
@@ -104,4 +157,73 @@ function tripal_bulk_loader_tripal_bulk_loader_analysis_related_tables () {
       'analysis_feature' => 'Features',
     ),
   );
-}
+}
+
+/**
+ * Implements hook_tripal_bulk_loader_<base table>_related_tables()
+ * This is used to supply the bulk loader with a list of tables and their 
+ * relationship to the base table.
+ *
+ * @return
+ * An array of relationships (any combination of 'Base Table', 'Foreign Key Relations', 
+ * 'Direct Relations', 'Indirect Relations') where each relationship is keyed by the type
+ * of relationship and the value is an array of tables with that relationship to the base table.
+ * Each table in the array of tables with that relationship should be of the form
+ * <chado table> => <Human-readable Name>.
+ */
+function tripal_bulk_loader_tripal_bulk_loader_stock_related_tables () {
+
+  return array(
+    'Base Table' => array(
+      'stock' => 'Stock'
+    ),
+    'Foreign Key Relations' => array(
+      'organism' => 'Organism',
+      'cvterm' => 'Type: Cv Term',
+      'dbxref' => 'Database Reference',
+    ),
+    'Direct Relations' => array(
+      'stockprop' => 'Stock Properties',
+      'stock_relationship' => 'Stock Relationships',
+    ),
+    'Indirect Relations' => array(
+      'stock_cvterm' => 'Additional Cv Terms',
+      'stock_dbxref' => 'Additional Database References',
+      'stock_genotype' => 'Genotypes',
+      'stock_pub' => 'Publications',
+      'stockcollection_stock' => 'Stock Collections'
+    ),
+  );
+
+}
+
+/**
+ * Implements hook_tripal_bulk_loader_<base table>_related_tables()
+ * This is used to supply the bulk loader with a list of tables and their 
+ * relationship to the base table.
+ *
+ * @return
+ * An array of relationships (any combination of 'Base Table', 'Foreign Key Relations', 
+ * 'Direct Relations', 'Indirect Relations') where each relationship is keyed by the type
+ * of relationship and the value is an array of tables with that relationship to the base table.
+ * Each table in the array of tables with that relationship should be of the form
+ * <chado table> => <Human-readable Name>.
+ */
+function tripal_bulk_loader_tripal_bulk_loader_genotype_related_tables () {
+
+  return array(
+    'Base Table' => array(
+      'genotype' => 'Genotype'
+    ),
+    'Indirect Relations' => array(
+      'feature_genotype' => 'Features',
+      'nd_experiment_genotype' => 'ND Experiments',
+      'phendesc' => 'Phenotypic Statements',
+      'phenotype_comparison' => 'Phenotype Comparisons',
+      'phenstatement' => 'Phenotypic Statements',
+      'stock_genotype' => 'Stocks',
+    ),
+  );
+
+}
+

tripal_bulk_loader/tripal_bulk_loader.loader.inc

@@ -38,7 +38,10 @@ function tripal_bulk_loader_add_loader_job_form_submit ($form, $form_state) {
 		$job_args[1] = $form_state['values']['nid'];
 		if (is_readable($form_state['values']['file'])) {
 			$fname = basename($form_state['values']['file']);
-			tripal_add_job("Bulk Loading Job: $fname",'tripal_bulk_loader', 'tripal_bulk_loader_load_data', $job_args, $user->uid);
+			$job_id = tripal_add_job("Bulk Loading Job: $fname",'tripal_bulk_loader', 'tripal_bulk_loader_load_data', $job_args, $user->uid);
+			
+			// add job_id to bulk_loader node
+			
 		} else {
 			drupal_set_message("Can not open ".$form_state['values']['file'].". Job not scheduled.");
 		}    
@@ -64,63 +67,38 @@ function tripal_bulk_loader_load_data($nid) {
   print "File: ".$node->file."\n";
   
   // Prep Work ==================================================================================
-  // get base table and it's table_description array
-  $base_table = $node->template->template_array['module'];
-  $base_table_desc = module_invoke_all('chado_'.$base_table.'_schema');
-  $related_tables = module_invoke_all('tripal_bulk_loader_'.$base_table.'_related_tables');
-  //print "Base Table: ".print_r($base_table,TRUE)."\nTable DEscription: ".print_r($base_table_desc, TRUE)."\nTemplate array: ".print_r($node->template->template_array,TRUE)."\n";
-
-  // get a default values array to be passed into tripal_core_chado_insert
-  // and get a mapping between table.field and spreadsheet column
-  // also determine relationship between each table and base table
-  $default_values_array = array();
+  // Generate default values array
+  $default_data = array();
   $field2column = array();
-  $relationships2base = array();
-  $all_tables = array();
-  foreach ($node->template->template_array as $table => $table_groups) {
-  	foreach ($table_groups as $group => $table_array) {
-			if (is_array($table_array)) {
-				$all_tables[$table] = $table;
-			
-				foreach ($table_array['field'] as $field_array) {
-					if (preg_match('/table field/', $field_array['type'])) {
-						$default_values_array[$table][$group][$field_array['field']] = '';
-						$field2column[$table][$group][$field_array['field']] = $field_array['spreadsheet column'];
-					} elseif (preg_match('/constant/', $field_array['type'])) {
-						$default_values_array[$table][$group][$field_array['field']] = $field_array['constant value'];
-					} else {
-						print 'WARNING: Unsupported type: '. $field_array['type'] . ' for ' . $table . '.' . $field_array['field']."!\n";
-					}
-				}
-				
-				// Determine what relation is between this table and the base table-----------------------
-				// This is used later to link the various records
-				
-				// Is there a foreign key to this table in the base table? 
-				if ($related_tables['Foreign Key Relations'][$table]) {
-						$relationships2base['foreign key'][$table] = $base_table_desc['foreign keys'][$table];
-				}
-				
-				// Is there a foreign key in this table to the base table? 
-				// ie: featureloc.feature_id (current) for feature (base)
-				if ($related_tables['Direct Relations'][$table]) {
-						$table_desc = module_invoke_all('chado_'.$table.'_schema');
-						$relationships2base['direct'][$table] = $table_desc['foreign keys'][$base_table];
-				}
-				
-				// Is there a linking table which links this table with the base table
-				if ($related_tables['Indirect Relations'][$table]) {
-					$table_desc = module_invoke_all('chado_'.$table.'_schema');
-					$relationships2base['indirect'][$table] = $table_desc['foreign keys'];
-				}
-	
-			}
-		}
-  } //end of preprocessing
-  
-  //print "\nDefault Values Array: ".print_r($default_values_array, TRUE)."\n";
+  $record2priority = array();
+  foreach ($node->template->template_array as $priority => $record_array) {
+    if (!is_array($record_array)) { continue; }
+    
+    foreach ($record_array['fields'] as $field_index => $field_array) {
+      $default_data[$priority]['table'] = $record_array['table'];
+      $record2priority[$record_array['record_id']] = $priority;
+      
+      if (preg_match('/table field/', $field_array['type'])) {
+        $default_data[$priority]['values_array'][$field_array['field']] = '';
+        $default_data[$priority]['need_further_processing'] = TRUE;
+        $field2column[$priority][$field_array['field']] = $field_array['spreadsheet column'];
+        
+      } elseif (preg_match('/constant/', $field_array['type'])) {
+        $default_data[$priority]['values_array'][$field_array['field']] = $field_array['constant value'];
+        
+      } elseif (preg_match('/foreign key/', $field_array['type'])) {
+        $default_data[$priority]['values_array'][$field_array['field']] = array();
+        $default_data[$priority]['values_array'][$field_array['field']]['foreign record'] = $field_array['foreign key'];
+        $default_data[$priority]['need_further_processing'] = TRUE;
+        
+      } else {
+        print 'WARNING: Unsupported type: '. $field_array['type'] . ' for ' . $table . '.' . $field_array['field']."!\n";
+      }    
+    } // end of foreach field
+  } //end of foreach record
+
+  //print "\nDefault Values Array: ".print_r($default_data, TRUE)."\n";
   //print "\nField to Column Mapping: ".print_r($field2column, TRUE)."\n";
-  //print "\nRelationships to Base Table: ".print_r($relationships2base, TRUE)."\n";
   
   // Parse File adding records as we go ========================================================
   $file_handle = fopen($node->file, 'r');
@@ -131,129 +109,46 @@ function tripal_bulk_loader_load_data($nid) {
   while (!feof($file_handle)) {
     $line = array();
     $raw_line = fgets($file_handle, 4096);
+    $raw_line = trim($raw_line);
     $line = preg_split("/\t/", $raw_line);
     $num_lines++;
+
+    $data = $default_data;
     
-    // Contains constants set above
-    $tables = $all_tables;
-    $values = $default_values_array;
-    
-    // Insert base record-----------------------------------------------------------------------
-    foreach ($relationships2base['foreign key'] as $table => $desc) {
-  		// check there is only 1 group & if more than one use first and warn
-  		if (sizeof($values[$table]) > 1) {
-  			print "WARNING: ".$table." has more than one group. There can only be one group for a "
-  				."table related to the base by a foreign key. Only the first will be used.\n";
-  		}
-  		
-      // Ensure foreign key is present
-      $group = key($values[$table]);
-      $sub_values = $values[$table][$group];
-      $sub_values = tripal_bulk_loader_supplement_values_array ($sub_values, $line, $field2column[$table][$group]);
-      $values[$table][$group] = $sub_values;
-      $success = tripal_bulk_loader_ensure_record ($table, $sub_values);   
-      if ($success) {
-        $num_records++;
-      } else {
-        $num_errors++;
-      }
-      
-      //Add to base values array
-      foreach ($desc['columns'] as $foreign_key => $primary_key) {
-				// check there is only 1 group & if more than one use first and warn
-				if (sizeof($values[$base_table]) > 1) {
-					print "WARNING: ".$table." has more than one group. There can only be one group for a "
-						."table related to the base by a foreign key. Only the first will be used.\n";
-				}   
-				$base_group = key($values[$base_table]);
-        $values[$base_table][$base_group][$foreign_key] = $values[$table][$group];
+    foreach ($data as $priority => $table_data) {
+      $table = $table_data['table'];
+      $values = $table_data['values_array'];
+      if ($table_data['need_further_processing']) {
+        $values = tripal_bulk_loader_add_spreadsheetdata_to_values ($values, $line, $field2column[$priority]);
+        $values = tripal_bulk_loader_add_foreignkey_to_values($values, $data, $record2priority);
       }
       
-      unset($tables[$table]);
-    }
-    
-    // base table
-    $values[$base_table][$base_group] = tripal_bulk_loader_supplement_values_array ($values[$base_table][$base_group], $line, $field2column[$base_table][$base_group]);
-    $success = tripal_bulk_loader_ensure_record ($base_table, $values[$base_table][$base_group]);     
-    //print 'Base Table values array: '.print_r($values[$base_table], TRUE)."\n";
-    if (!$success) {
-      print "ERROR: Unable to insert base record where base table: ".$base_table.
-        " and values array: ".tripal_bulk_loader_flatten_array($values[$base_table][$base_group])."\n";
-      $num_errors++;
-    } else {
-      $num_records++;
-    }
-    unset($tables[$base_table]);
-    
-    // Insert all tables with direct relationship to base ----------------------------------------
-    foreach ($relationships2base['direct'] as $table => $desc) {
-			foreach ($values[$table] as $group => $sub_values) {
-	      //Add base to values array
-  	    foreach ($desc['columns'] as $foreign_key => $primary_key) {  	
-        	$values[$table][$group][$foreign_key] = $values[$base_table][$base_group];
+      // add new values array into the data array
+      $data[$priority]['values_array'] = $values;
+       
+      // first check if it already exists
+      $exists = tripal_core_chado_select($table, array_keys($values), $values, array('has_record'=>TRUE));
+      if ($exists) {
+          watchdog('T_bulk_loader', 
+            'Record already exists in %table: %record',
+            array('%table' => $table, '%record' => tripal_bulk_loader_flatten_array($values)),
+            'WATCHDOG_WARNING'
+          );      
+      } else {
+        // if it doesn't exist already then insert it
+        $success = tripal_core_chado_insert($table, $values);
+        if (!$success) {
+          watchdog('T_bulk_loader', 
+            'Unable to insert the following record into %table: %record',
+            array('%table' => $table, '%record' => tripal_bulk_loader_flatten_array($values)),
+            'WATCHDOG_ERROR'
+          );
         }
+      }// end of if/not record exists
+    } // end of foreach table in default data array
 
-				// Supplement and Add record
-				$values[$table][$group] = tripal_bulk_loader_supplement_values_array ($values[$table][$group], $line, $field2column[$table][$group]);
-				$success = tripal_bulk_loader_ensure_record ($table, $values[$table][$group]);
-				if ($success) {
-					$num_records++;
-				} else {
-					$num_errors++;
-				}
-				unset($tables[$table]);
-			}
-    } 
-    
-    // Add in all other tables -----------------------------------------------------------------
-    foreach ($tables as $table) {
-      // Don't insert if its an indirect relationship linking table
-      if (!$relationships2base['indirect'][$table]) {
-      	foreach ($values[$table] as $group => $sub_values) {
-					// Supplement and Add record
-					$values[$table][$group] = tripal_bulk_loader_supplement_values_array ($values[$table][$group], $line, $field2column[$table][$group]);
-					$success = tripal_bulk_loader_ensure_record ($table, $values[$table][$group]);    
-					if ($success) {
-						$num_records++;
-					} else {
-						$num_errors++;
-					}
-				}
-      }
-    }
-    
-    // Add in indirect relationships -----------------------------------------------------------
-    foreach ($relationships2base['indirect'] as $table => $desc) {
-    	foreach ($values[$table] as $group => $sub_values) {
-				// Add foreign keys to values array
-				foreach ($desc as $subtable => $subdesc) {
-					foreach ($subdesc['columns'] as $foreign_key => $primary_key) {
-						$values[$table][$group][$foreign_key] = $values[$subtable][$group];
-					}
-				}
-				
-				// Supplement and Add record
-				$values[$group][$table] = tripal_bulk_loader_supplement_values_array ($values[$table][$group], $line, $field2column[$table][$group]);
-				$success = tripal_bulk_loader_ensure_record ($table, $values[$table][$group]);      
-				if ($success) {
-					$num_records++;
-				} else {
-					$num_errors++;
-				}
-			}
-		}
-		
-  } //end of file
+  } //end of foreach line of file
   
-  print "\nNumber of Records Inserted:".$num_records."\n";
-  print "Number of Lines in File:".$num_lines."\n";
-  print "Number of Errors: ".$num_errors."\n\n";
-  
-  if (!$num_errors) {
-    db_query("UPDATE {tripal_bulk_loader} SET job_status='%s' WHERE nid=%d", 'successfully loaded', $nid);
-  } else {
-    db_query("UPDATE {tripal_bulk_loader} SET job_status='%s' WHERE nid=%d", 'load attempted', $nid);
-  }
 }
 
 /**
@@ -268,9 +163,11 @@ function tripal_bulk_loader_load_data($nid) {
  * @return 
  *   Supplemented values array
  */
-function tripal_bulk_loader_supplement_values_array ($values, $line, $field2column) {
+function tripal_bulk_loader_add_spreadsheetdata_to_values ($values, $line, $field2column) {
 
   foreach ($values as $field => $value) {
+    if (is_array($value)) { continue; }
+    
     $column = $field2column[$field] - 1;
     if ($line[$column] OR (!$values[$field])) {
       $values[$field] = $line[$column];
@@ -281,36 +178,27 @@ function tripal_bulk_loader_supplement_values_array ($values, $line, $field2colu
 }
 
 /**
- * This function first ensures the record doesn't already exist and then inserts it
+ * Handles foreign keys in the values array.
+ *
+ * Specifically, if the value for a field is an array then it is assumed that the array contains
+ * the name of the record whose values array should be substituted here. Thus the foreign
+ * record is looked up and the values array is substituted in.
  *
- * @param $table
- *   The table the record should be present in
- * @param $values
- *   The values array used for selecting and/or inserting the record
- * @return
- *   TRUE or FALSE based on presence or absence of record
  */
-function tripal_bulk_loader_ensure_record ($table, $values) {
-  
-  // get flattened values array for printing errors
-  $flattened_values = tripal_bulk_loader_flatten_array($values);
-  
-  // check if record exists
-  $has_record = tripal_core_chado_select($table, array_keys($values), $values, array('has_record' => TRUE));
-  if ($has_record) {
-    print "\tWARNING: Record already exists in $table where ".implode(',',$flattened_values).".\n";
-    return true;
-  } else {
-    // if record doesn't exist then insert it
-    $success = tripal_core_chado_insert($table, $values);
-    if (!$success) {
-      print "ERROR: Unable to insert the following record into $table: ".implode(',',$flattened_values)."\n";
-      return false;
-    } else {
-      return true;
+function tripal_bulk_loader_add_foreignkey_to_values($values, $data, $record2priority) {
+
+  foreach ($values as $field => $value) {
+    if (is_array($value)) {
+      $foreign_record = $value['foreign record'];
+      $foreign_priority = $record2priority[$foreign_record];
+      $foreign_values = $data[$foreign_priority]['values_array'];
+      
+      //add to current values array
+      $values[$field] = $foreign_values;
     }
   }
-      
+
+  return $values;
 }
 
 /**

tripal_bulk_loader/tripal_bulk_loader.module

@@ -42,6 +42,51 @@ function tripal_bulk_loader_menu() {
       'type' => MENU_NORMAL_ITEM,   
 		'file' => 'tripal_bulk_loader.admin.inc',
 	);
+	// Edit Template------
+	$items['admin/tripal/tripal_bulk_loader_template/edit'] = array(
+      'title' => 'Edit Bulk Loader Template',
+      'description' => 'Create loader template for loading tab-delimited data',
+      'page callback' => 'drupal_get_form',
+      'page arguments' => array('tripal_bulk_loader_edit_template_base_form'),
+      'access arguments' => array('administer site configuration'),
+      'type' => MENU_NORMAL_ITEM,   
+		'file' => 'tripal_bulk_loader.admin.inc',
+	);	
+	$items['admin/tripal/tripal_bulk_loader_template/edit/add_field'] = array(
+      'title' => 'Add Template Field',
+      'description' => 'Add a template field to an existing tripal bulk loader template.',
+      'page callback' => 'drupal_get_form',
+      'page arguments' => array('tripal_bulk_loader_add_template_field_form'),
+      'access arguments' => array('administer site configuration'),
+      'type' => MENU_CALLBACK,   
+		'file' => 'tripal_bulk_loader.admin.inc',
+	);
+	// Delete Template -----
+	$items['admin/tripal/tripal_bulk_loader_template/delete'] = array(
+      'title' => 'Delete Bulk Loader Template',
+      'description' => 'Delete bulk loader template',
+      'page callback' => 'drupal_get_form',
+      'page arguments' => array('tripal_bulk_loader_delete_template_base_form'),
+      'access arguments' => array('administer site configuration'),
+      'type' => MENU_NORMAL_ITEM,   
+		'file' => 'tripal_bulk_loader.admin.inc',
+	);
+	// AHAH ---------
+	$items['admin/tripal/tripal_bulk_loader_template/add_field_ahah'] = array(
+      'page callback' => 'tripal_bulk_loader_add_field_ahah',
+      'access arguments' => array('administer site configuration'),
+      'type' => MENU_CALLBACK,   
+		  'file' => 'tripal_bulk_loader.admin.inc',	
+	);
+	
+	// OLD AHAH-----
+	$items['admin/tripal/tripal_bulk_loader_template/field_extra_options_ahah'] = array(
+      'page callback' => 'tripal_bulk_loader_field_extra_options_ahah',
+      'access arguments' => array('administer site configuration'),
+      'type' => MENU_CALLBACK,   
+		  'file' => 'tripal_bulk_loader.admin.inc',
+	);	
+	// Add Field AHAH
 	$items['admin/tripal/tripal_bulk_loader_template/add/chado_column_ahah'] = array(
       'page callback' => 'tripal_bulk_loader_add_chado_column_ahah',
       'access arguments' => array('administer site configuration'),
@@ -54,22 +99,19 @@ function tripal_bulk_loader_menu() {
       'type' => MENU_CALLBACK,   
 		  'file' => 'tripal_bulk_loader.admin.inc',
 	);
-	//tripal_bulk_loader_edit_template_base_form
-	$items['admin/tripal/tripal_bulk_loader_template/edit'] = array(
-      'title' => 'Edit Bulk Loader Template',
-      'description' => 'Edit bulk loader template',
-      'page callback' => 'drupal_get_form',
-      'page arguments' => array('tripal_bulk_loader_edit_template_base_form'),
-      'access arguments' => array('administer site configuration'),
-      'type' => MENU_NORMAL_ITEM,   
-			'file' => 'tripal_bulk_loader.admin.inc',
-	);	
-	$items['admin/tripal/tripal_bulk_loader_template/edit/edit_fields_ahah'] = array(
-      'page callback' => 'tripal_bulk_loader_edit_fields_ahah',
+	$items['admin/tripal/tripal_bulk_loader_template/add/record_group_ahah'] = array(
+      'page callback' => 'tripal_bulk_loader_add_new_record_ahah',
       'access arguments' => array('administer site configuration'),
       'type' => MENU_CALLBACK,   
 		  'file' => 'tripal_bulk_loader.admin.inc',
 	);
+
+	$items['admin/tripal/tripal_bulk_loader_template/edit/change_subform_ahah'] = array(
+      'page callback' => 'tripal_bulk_loader_edit_change_subform_ahah',
+      'access arguments' => array('administer site configuration'),
+      'type' => MENU_CALLBACK,   
+		  'file' => 'tripal_bulk_loader.admin.inc',
+	);	
 	$items['admin/tripal/tripal_bulk_loader_template/edit/chado_column_ahah'] = array(
       'page callback' => 'tripal_bulk_loader_edit_chado_column_ahah',
       'access arguments' => array('administer site configuration'),
@@ -82,15 +124,7 @@ function tripal_bulk_loader_menu() {
       'type' => MENU_CALLBACK,   
 		  'file' => 'tripal_bulk_loader.admin.inc',
 	);
-	$items['admin/tripal/tripal_bulk_loader_template/delete'] = array(
-      'title' => 'Delete Bulk Loader Template',
-      'description' => 'Delete bulk loader template',
-      'page callback' => 'drupal_get_form',
-      'page arguments' => array('tripal_bulk_loader_delete_template_base_form'),
-      'access arguments' => array('administer site configuration'),
-      'type' => MENU_NORMAL_ITEM,   
-		'file' => 'tripal_bulk_loader.admin.inc',
-	);
+	
   return $items;
 }
 
@@ -308,6 +342,9 @@ function tripal_bulk_loader_update ($node) {
 	}
 }
 
+/**
+ * Preprocessor function for the tripal_bulk_loader template
+ */
 function tripal_bulk_loader_preprocess_tripal_bulk_loader_template (&$variables) {
 
 	$sql = "SELECT * FROM {tripal_bulk_loader_template} WHERE template_id=%d";
@@ -315,4 +352,28 @@ function tripal_bulk_loader_preprocess_tripal_bulk_loader_template (&$variables)
 	$template->template_array = unserialize($template->template_array);	
 	$variables['template'] = $template;
 	
+}
+
+/**
+ * Implements hook_job_describe_args()
+ * Specifically to make viewing past tripal jobs more readable for jobs registered by this module
+ *
+ * @params $callback
+ *   The callback passed into tripal_add_job()
+ * @param $args
+ *   The arguements passed into tripal_add_job()
+ * @return 
+ *   An array where keys are the human readable headers describing each arguement 
+ *   and the value is the aguement passed in after formatting
+ */
+function tripal_bulk_loader_job_describe_args($callback,$args){
+
+  $new_args = array();
+  if($callback == 'tripal_bulk_loader_load_data'){
+    //1st arg is the nid for a bulk loader node
+    $node = node_load($args[0]);
+    $new_args['Bulk Loading Job'] = l($node->title, 'node/'.$args[0]);
+    return $new_args;
+  }
+  
 }

tripal_core/jobs.php

@@ -29,7 +29,7 @@ function tripal_add_job ($job_name,$modulename,$callback,$arguments,$uid,$priori
       drupal_set_message("Failed to add job $job_name.");
    }
 
-   return 1;
+   return $record->job_id;
 }
 /**
 *   

tripal_core/tripal_core.api.inc

@@ -904,11 +904,6 @@ function tripal_core_generate_chado_var($table, $values) {
     return $results;
   } else {
     // no results returned
-    watchdog('tripal_core',
-      'tripal_core_generate_chado_var for %table: No record matches criteria values:%values',
-      array('%table'=>$table, '%values'=>print_r($values,TRUE)),
-      WATCHDOG_ERROR
-    );  
   } 
   
 }
@@ -1096,12 +1091,12 @@ function tripal_core_expand_chado_vars ($object, $type, $to_expand) {
     } else {
       // if there is an expandable array then we've reached the base object
       // if we get here and don't have anything expanded then something went wrong
-      watchdog(
-        'tripal_core',
-        'tripal_core_expand_chado_vars: Unable to expand the %type %to_expand',
-        array('%type'=>$type, '%to_expand'=>$to_expand),
-        WATCHDOG_ERROR
-      );
+//      watchdog(
+//        'tripal_core',
+//        'tripal_core_expand_chado_vars: Unable to expand the %type %to_expand',
+//        array('%type'=>$type, '%to_expand'=>$to_expand),
+//        WATCHDOG_ERROR
+//      );
     } //end of it we've reached the base object
   }
   

tripal_core/tripal_core.schema.api.inc

@@ -6,6 +6,28 @@
  *        (specifically to add missing foreign key definitions) by implementing
  *        hook_chado_<table name>_schema().
  */
+
+/**
+ *
+ */
+function tripal_core_get_chado_tables() {
+  if(is_array($db_url) and array_key_exists('chado',$db_url)){
+    $previous_db = tripal_db_set_active('chado');
+    $sql = 'SELECT tablename FROM pg_tables';
+    $resource = db_query($sql);
+    tripal_db_set_active($previous_db);
+  } else {
+    $sql = "SELECT tablename FROM pg_tables WHERE schemaname='chado'";
+    $resource = db_query($sql);
+  }
+  
+  $tables = array();
+  while ($r = db_fetch_object($resource)) {
+    $tables[$r->tablename] = $r->tablename;
+  }
+  asort($tables);
+  return $tables;
+}
  
 /**
  * Implements hook_organism_dbxref_schema()

tripal_genetic/tripal_genetic.api.inc

@@ -0,0 +1,66 @@
+<?php 
+
+/**
+ * Implements hook_chado_genotype_schema()
+ * Purpose: To add descriptions and foreign keys to default table description
+ * Note: This array will be merged with the array from all other implementations
+ *
+ * @return
+ *    Array describing the genotype table
+ *
+ * @ingroup tripal_schema_api
+ */
+function tripal_genetic_chado_genotype_schema () {
+  $description = array();
+ 
+  $referring_tables = array('analysisfeature',
+    'feature_genotype',
+    'phendesc',
+    'phenotype_comparison',
+    'phenstatement',
+    'stock_genotype',
+  );
+  $description['referring_tables'] = $referring_tables;
+  
+  return $description;
+  
+}
+
+/**
+ * Implements hook_chado_genotype_schema()
+ * Purpose: To add descriptions and foreign keys to default table description
+ * Note: This array will be merged with the array from all other implementations
+ *
+ * @return
+ *    Array describing the genotype table
+ *
+ * @ingroup tripal_schema_api
+ */
+function tripal_genetic_chado_feature_genotype_schema () {
+  $description = array();
+
+  $description['foreign keys']['feature'] = array(
+        'table' => 'feature',
+        'columns' => array(
+          'feature_id' => 'feature_id',
+          'chromosome_id' => 'feature_id',
+        ),
+  ); 
+
+  $description['foreign keys']['genotype'] = array(
+        'table' => 'genotype',
+        'columns' => array(
+          'genotype_id' => 'genotype_id',
+        ),
+  );  
+
+  $description['foreign keys']['cvterm'] = array(
+        'table' => 'cvterm',
+        'columns' => array(
+          'type_id' => 'cvterm_id',
+        ),
+  );
+  
+  return $description;
+  
+}

tripal_genetic/tripal_genetic.module

@@ -9,6 +9,8 @@
  * @see http://gmod.org/wiki/Chado_General_Module
  */
 
+require('tripal_genetic.api.inc');
+
 /*************************************************************************
  * Implements hook_views_api()
  * Purpose: Essentially this hook tells drupal that there is views support for

tripal_stock/tripal_stock.api.inc

@@ -199,7 +199,7 @@ function tripal_stock_get_stock_by_name_identifier($name, $organism_id) {
   $stock_ids = array();
   
   $options = array(
-    'regex_columns' => array('name', 'uniquename', 'accession', 'value')
+    'case_insensitive_columns' => array('name', 'uniquename', 'accession', 'value')
   );
   
   // where name_identifier = stock.name-------------------------------
@@ -209,7 +209,7 @@ function tripal_stock_get_stock_by_name_identifier($name, $organism_id) {
       'organism_id' => $organism_id,
     ),
     array(
-      'regex_columns' => array('name'),
+      'case_insensitive_columns' => array('name'),
     )
   );
   if (!empty($current_stocks)) {
@@ -223,7 +223,7 @@ function tripal_stock_get_stock_by_name_identifier($name, $organism_id) {
       'organism_id' => $organism_id,
     ),
     array(
-      'regex_columns' => array('uniquename'),
+      'case_insensitive_columns' => array('uniquename'),
     )
   );
   if (!empty($current_stocks)) {
@@ -240,7 +240,7 @@ function tripal_stock_get_stock_by_name_identifier($name, $organism_id) {
       'organism_id' => $organism_id,
     ),
     array(
-      'regex_columns' => array('accession'),
+      'case_insensitive_columns' => array('accession'),
     )
   );
   if (!empty($current_stocks)) {
@@ -258,7 +258,7 @@ function tripal_stock_get_stock_by_name_identifier($name, $organism_id) {
       ),
     ),
     array(
-      'regex_columns' => array('accession'),
+      'case_insensitive_columns' => array('accession'),
     )
   );
   if (!empty($current_stocks)) {
@@ -279,7 +279,7 @@ function tripal_stock_get_stock_by_name_identifier($name, $organism_id) {
       'value' => $name,
     ),
     array(
-      'regex_columns' => array('value'),
+      'case_insensitive_columns' => array('value'),
     )
   );
   if (!empty($current_stocks)) {