فهرست منبع

Updated Bulk Loader to allow for using non FK fields when referring to another record.

spficklin 12 سال پیش
والد
کامیت
f937c72093
2فایلهای تغییر یافته به همراه334 افزوده شده و 134 حذف شده
  1. 247 114
      tripal_bulk_loader/tripal_bulk_loader.admin.templates.inc
  2. 87 20
      tripal_bulk_loader/tripal_bulk_loader.loader.inc

+ 247 - 114
tripal_bulk_loader/tripal_bulk_loader.admin.templates.inc

@@ -218,6 +218,28 @@ function tripal_bulk_loader_modify_template_base_form($form_state = NULL, $mode)
         );
 
         foreach ($table_array['fields'] as $field_index => $field) {
+          $fk_value = '';
+          if ($field['foreign key']) {
+            if ($field['foreign field']) {
+              $fk_value = $field['foreign key'] . " (" . $field['foreign field'] . ")";
+            }
+            else {
+              // for backwards compatibility we need to get the FK relationship to find
+              // out what field we're joining on.  For templates created using a 
+              // previous version this information isn't stored in the template 
+              // so we need to get it.
+              $fk_priority = $form_state['storage']['record2priority'][$field['foreign key']];
+              $fk_table = $form_state['storage']['template'][$fk_priority]['table'];    
+              $tbl_description = tripal_core_get_chado_table_schema($table_array['table']);
+              foreach ($tbl_description['foreign keys'] as $key_table => $key_array) {
+                foreach ($key_array['columns'] as $left_field => $right_field) {
+                  if($key_table == $fk_table and $left_field == $field['field']){                  
+                    $fk_value = $field['foreign key'] . " ($right_field)";
+                  }
+                }
+              }
+            }            
+          }
 
           $form['fields']['fields-data'][$i] = array(
             'record_id' => array(
@@ -262,7 +284,7 @@ function tripal_bulk_loader_modify_template_base_form($form_state = NULL, $mode)
             ),
             'foreign_record_id' => array(
               '#type' => 'item',
-              '#value' => $field['foreign key'],
+              '#value' => $fk_value,
             ),
             'edit_submit' => array(
               '#type' => 'submit',
@@ -932,12 +954,12 @@ function tripal_bulk_loader_add_template_field_form(&$form_state = NULL) {
     $form_state['storage']['template'] = $template;
 
     $form_state['storage']['record2priority'] = array();
-  foreach ($form_state['storage']['template_array'] as $priority => $record_array) {
-    if (!is_array($record_array)) {
-      continue;
+    foreach ($form_state['storage']['template_array'] as $priority => $record_array) {
+      if (!is_array($record_array)) {
+        continue;
+      }
+      $form_state['storage']['record2priority'][$record_array['record_id']] = $priority;
     }
-    $form_state['storage']['record2priority'][$record_array['record_id']] = $priority;
-  }
 
     $form_state['storage']['referring URL'] = $_SERVER["HTTP_REFERER"];
   }
@@ -967,7 +989,7 @@ function tripal_bulk_loader_add_template_field_form(&$form_state = NULL) {
     $table = reset($tables);
   }
 
-   // get the record_id from the path
+  // get the record_id from the path
   if ($_GET['record_id'] !== NULL) {
     $form_state['values']['field_group'] = $_GET['record_id'];
     if (preg_match('/\d+/', $_GET['record_id'])) {
@@ -975,22 +997,31 @@ function tripal_bulk_loader_add_template_field_form(&$form_state = NULL) {
       $table = $form_state['storage']['template_array'][$priority]['table'];
     }
   }
-
-  // Fields and foreign key mappings
+  
+  $show_all = $form_state['values']['show_all_records'];
+  
+  $table_description = tripal_core_get_chado_table_schema($table);
+  
+  // Fields and foreign key / referral mappings
+  // build the fields array
   $chado_fields = array();
+  foreach ($table_description['fields'] as $field_name => $field_array) {
+    $chado_fields[$field_name] = $field_name;
+  }  
+
+  $ref_chado_fields = array();
   $fk_options = array();
   $fk_options['NULL'] = 'None';
-  $table_description = tripal_core_get_chado_table_schema($table);
-  //dpm($table_description, 'table description for |'.$table.'|');
-  if ($field_type == 'foreign key') {
+  if ($field_type == 'foreign key' and !$show_all) {
+
     $foreign_field2table = array();
     foreach ($table_description['foreign keys'] as $key_table => $key_array) {
       foreach ($key_array['columns'] as $left_field => $right_field) {
-        $chado_fields[$left_field] = $left_field;
+        //$chado_fields[$left_field] = $left_field;
         $foreign_field2table[$left_field] = $key_table;
       }
     }
-    reset($chado_fields);
+//   reset($chado_fields);
 
     // set default field
     if (empty($chado_fields)) {
@@ -1001,7 +1032,7 @@ function tripal_bulk_loader_add_template_field_form(&$form_state = NULL) {
     }
     else {
       $field = current($chado_fields);
-    }
+    } 
 
     // Foreign key options
     $foreign_table = $foreign_field2table[$field];
@@ -1013,29 +1044,40 @@ function tripal_bulk_loader_add_template_field_form(&$form_state = NULL) {
       }
     }
   }
-  else {
-    foreach ($table_description['fields'] as $field_name => $field_array) {
-        $chado_fields[$field_name] = $field_name;
+  // the user wants to see all of the records
+  elseif ($field_type == 'foreign key' and $show_all) {
+    foreach ($form_state['storage']['record2priority'] as $record_name => $priority ) {
+      $fk_options[$record_name] = $record_name;
+    }
+  }
+    
+  // build the list of referrer table fields
+  if ($field_type == 'foreign key') {
+    $fk_rec = $form_state['values']['foreign_record']; 
+    if ($fk_rec and $fk_rec != 'None' and $fk_rec != 'NULL') {
+    
+      // first add in the primary keys
+      $fk_priority = $form_state['storage']['record2priority'][$fk_rec];
+      $fk_table = $form_state['storage']['template_array'][$fk_priority]['table'];
+      foreach ($table_description['foreign keys'] as $key_table => $key_array) {
+        foreach ($key_array['columns'] as $left_field => $right_field) {
+          if($key_table == $fk_table and $left_field == $field){                  
+            $ref_chado_fields['Foreign Key'][$right_field] = $right_field;
+          }
+        }
+      }      
+      $fk_description = tripal_core_get_chado_table_schema($fk_table);
+      foreach ($fk_description['fields'] as $fk_field_name => $fk_farray){
+         // don't include the FK field it's included above
+         if (in_array($fk_field_name,$ref_chado_fields['Foreign Key'])) {
+           continue;
+         }
+         $ref_chado_fields['Additional Table Fields'][$fk_field_name] = $fk_field_name;
+      }
     }
   }
 
-  $variables = array(
-    'form_state' => $form_state,
-    'tables' => $tables,
-    'default table' => $table,
-    'fields' => $chado_fields,
-    'default_field' => $field,
-    'priority' => $priority,
-    'record_name' => $record_name,
-    'table description' => $table_description,
-    'foreign key options' => $fk_options,
-    'foreign field=>table' => $foreign_field2table,
-    'foreign table' => $foreign_table,
-  );
-  //dpm($variables, 'variables');
-
-  // Start of Form Proper--------------------------------------------------------------
-  //dpm($form_state, 'Form State');
+  // Start of Form Proper-------------------------------------------------------------
 
   $form['template_name'] = array(
     '#type' => 'item',
@@ -1060,7 +1102,7 @@ function tripal_bulk_loader_add_template_field_form(&$form_state = NULL) {
     '#options' => array(
       'table field' => t('Data: A Field which maps to a column in the supplied file.'),
       'constant' => t('Constant: Field which remains Constant throughout the file'),
-      'foreign key' => t('Foreign Key: Fields which map to a record in another table'),
+      'foreign key' => t('Record Referral: Fields which map to a record in another table'),
     ),
     '#required' => TRUE,
     '#default_value' => $field_type,
@@ -1109,6 +1151,38 @@ function tripal_bulk_loader_add_template_field_form(&$form_state = NULL) {
     '#default_value' => $form_state['values']['field_title'],
   );
 
+
+  // Chado Field
+  $form['add_fields']['chado'] = array(
+    '#type' => 'fieldset',
+    '#title' => t('Chado Field/Column Details'),
+    '#description' => t('Specify the Table/Field in chado that this field maps to.'),
+  );
+
+  $form['add_fields']['chado']['chado_table'] = array(
+    '#type' => 'select',
+    '#title' => t('Chado Table'),
+    '#options' => $tables,
+    '#default_value' => $table,
+    '#ahah' => array(
+      'path' => 'admin/tripal/tripal_bulk_loader_template/add_field_ahah',
+      'wrapper' => 'tripal_bulk_loader_template-add_field',
+      'effect' => 'fade'
+      ),
+  );
+
+  $form['add_fields']['chado']['chado_field'] = array(
+    '#type' => 'select',
+    '#title' => t('Chado Field/Column'),
+    '#options' => $chado_fields,
+    '#default_value' => $form_state['values']['chado_field'],
+    '#ahah' => array(
+      'path' => 'admin/tripal/tripal_bulk_loader_template/add_field_ahah',
+      'wrapper' => 'tripal_bulk_loader_template-add_field',
+      'effect' => 'fade'
+    ),
+  );
+  
   // loading file data column
   $form['add_fields']['columns'] = array(
     '#type' => 'fieldset',
@@ -1126,7 +1200,7 @@ function tripal_bulk_loader_add_template_field_form(&$form_state = NULL) {
     '#default_value' => ($form_state['values']['sheet_name'])? $form_state['values']['sheet_name'] : 'Sheet1',
   );
   */
-
+  
   $form['add_fields']['columns']['column_number'] = array(
     '#type' => 'textfield',
     '#title' => t('Column'),
@@ -1176,52 +1250,48 @@ function tripal_bulk_loader_add_template_field_form(&$form_state = NULL) {
     '#description' => t('Checks the database when a bulk loading job is created to ensure the value entered already exists in the database.'),
   );
 
-  // Foreign Key
+  // Foreign Key / Referrer
   $form['add_fields']['foreign_key'] = array(
     '#type' => 'fieldset',
-    '#title' => 'Foreign Key',
+    '#title' => 'Record Referral',
     '#collapsible' => TRUE,
     '#collapsed' => ($field_type == 'foreign key')? FALSE : TRUE,
   );
-
-  $form['add_fields']['foreign_key']['foreign_record'] = array(
-    '#type' => 'select',
-    '#title' => 'Record to refer to',
-    '#descripion' => 'Select the record that this foreign key shouold refer to. The record needs to already exist and be of the correct table.',
-    '#options' => $fk_options,
-  );
-
-  // Chado Field
-  $form['add_fields']['chado'] = array(
-    '#type' => 'fieldset',
-    '#title' => t('Chado Field/Column Details'),
-    '#description' => t('Specify the Table/Field in chado that this field maps to.'),
-  );
-
-  $form['add_fields']['chado']['chado_table'] = array(
-    '#type' => 'select',
-    '#title' => t('Chado Table'),
-    '#options' => $tables,
-    '#default_value' => $table,
+  
+  $form['add_fields']['foreign_key']['show_all_records'] = array(
+    '#type' => 'checkbox',
+    '#title' => 'Refer to any record',
+    '#description' => t('By default, the bulk loader will only allow referral to records in a foreign key relationship.  To allow referral to any previous record, check this box'),
+    '#default_value' => $show_all,
     '#ahah' => array(
       'path' => 'admin/tripal/tripal_bulk_loader_template/add_field_ahah',
       'wrapper' => 'tripal_bulk_loader_template-add_field',
       'effect' => 'fade'
-      ),
+    ),    
   );
-
-  $form['add_fields']['chado']['chado_field'] = array(
+  
+  $form['add_fields']['foreign_key']['foreign_record'] = array(
     '#type' => 'select',
-    '#title' => t('Chado Field/Column'),
-    '#options' => $chado_fields,
-    '#default_value' => $form_state['values']['chado_field'],
+    '#title' => 'Record to refer to',
+    '#descripion' => 'Select the record that this value should refer to. The record needs to already exist.',
+    '#options' => $fk_options,
     '#ahah' => array(
       'path' => 'admin/tripal/tripal_bulk_loader_template/add_field_ahah',
       'wrapper' => 'tripal_bulk_loader_template-add_field',
       'effect' => 'fade'
-    ),
+    ),  
+    '#default_value' => ($form_state['values']['foreign_record']) ? $form_state['values']['foreign_record'] : $template_field['foreign key'],
   );
-
+  
+  $form['add_fields']['foreign_key']['foreign_field'] = array(
+    '#type' => 'select',
+    '#title' => 'Field to refer to',
+    '#descripion' => 'Select the record that this value should refer to. The record needs to already exist.',
+    '#options' => $ref_chado_fields,
+    '#default_value' => ($form_state['values']['foreign_field']) ? $form_state['values']['foreign_field'] : $template_field['foreign_field'],
+  );
+  
+  
   $form['add_fields']['additional'] = array(
     '#type' => 'fieldset',
     '#title' => 'Additional Options',
@@ -1453,7 +1523,9 @@ function tripal_bulk_loader_add_template_field_form_submit($form, &$form_state)
           'type' => 'foreign key',
           'title' => $form_state['values']['field_title'],
           'field' => $form_state['values']['chado_field'],
+          'show_all_records' => $form_state['values']['show_all_records'],
           'foreign key' => $form_state['values']['foreign_record'],
+          'foreign field' => $form_state['values']['foreign_field'],
           'required' => $form_state['values']['required'],
         );
       }
@@ -1599,7 +1671,7 @@ function tripal_bulk_loader_edit_template_field_form(&$form_state = NULL) {
 
   $field_type = ($form_state['values']['field_type'])? $form_state['values']['field_type'] : $template_field['type'];
 
-  // Tables and default table
+  // Get the tables array and default table
   $tables = tripal_core_get_chado_tables(TRUE);
   if ($form_state['values']) {
     $table = $form_state['values']['chado_table'];
@@ -1608,21 +1680,29 @@ function tripal_bulk_loader_edit_template_field_form(&$form_state = NULL) {
     $table = $form_state['storage']['template_array'][$priority]['table'];
   }
 
+  $show_all = ($form_state['values']['show_all_records'] ? $form_state['values']['show_all_records'] : $template_field['show_all_records']);
+  $table_description = tripal_core_get_chado_table_schema($table);
 
-  // Fields and foreign key mappings
+  // Fields and foreign key / referral mappings
+  // build the fields array
   $chado_fields = array();
+  foreach ($table_description['fields'] as $field_name => $field_array) {
+    $chado_fields[$field_name] = $field_name;
+  }  
+
+  $ref_chado_fields = array();
   $fk_options = array();
   $fk_options['NULL'] = 'None';
-  $table_description = tripal_core_get_chado_table_schema($table);
-  if ($field_type == 'foreign key') {
+  if ($field_type == 'foreign key' and !$show_all) {
+  
     $foreign_field2table = array();
     foreach ($table_description['foreign keys'] as $key_table => $key_array) {
       foreach ($key_array['columns'] as $left_field => $right_field) {
-        $chado_fields[$left_field] = $left_field;
+        //$chado_fields[$left_field] = $left_field;
         $foreign_field2table[$left_field] = $key_table;
       }
     }
-    reset($chado_fields);
+//    reset($chado_fields);
 
     // set default field
     if (empty($chado_fields)) {
@@ -1637,7 +1717,6 @@ function tripal_bulk_loader_edit_template_field_form(&$form_state = NULL) {
     else {
       $field = current($chado_fields);
     }
-    //dpm($field, 'field');
 
     // Foreign key options
     $foreign_table = $foreign_field2table[$field];
@@ -1649,16 +1728,40 @@ function tripal_bulk_loader_edit_template_field_form(&$form_state = NULL) {
       }
     }
   }
-  else {
-    foreach ($table_description['fields'] as $field_name => $field_array) {
-        $chado_fields[$field_name] = $field_name;
+  // the user wants to see all of the records
+  elseif ($field_type == 'foreign key' and $show_all) {
+    foreach ($form_state['storage']['record2priority'] as $record_name => $priority ) {
+      $fk_options[$record_name] = $record_name;
+    }
+  }
+  
+  // build the list of referrer table fields
+  if ($field_type == 'foreign key') {
+    $fk_rec = $form_state['values']['foreign_record'] ?  $form_state['values']['foreign_record'] : $template_field['foreign key'];
+    if ($fk_rec and $fk_rec != 'None' and $fk_rec != 'NULL') {
+    
+      // first add in the foreign keys
+      $fk_priority = $form_state['storage']['record2priority'][$fk_rec];
+      $fk_table = $form_state['storage']['template_array'][$fk_priority]['table'];
+      foreach ($table_description['foreign keys'] as $key_table => $key_array) {
+        foreach ($key_array['columns'] as $left_field => $right_field) {
+          if($key_table == $fk_table and $left_field == $field){                  
+            $ref_chado_fields['Foreign Key'][$right_field] = $right_field;
+          }
+        }
+      }      
+      $fk_description = tripal_core_get_chado_table_schema($fk_table);
+      foreach ($fk_description['fields'] as $fk_field_name => $fk_farray){
+         // don't include the FK field it's included above
+         if (in_array('Foreign Key',$ref_chado_fields) and 
+             in_array($fk_field_name, $ref_chado_fields['Foreign Key'])) {
+           continue;
+         }
+         $ref_chado_fields['Additional Table Fields'][$fk_field_name] = $fk_field_name;
+      }
     }
   }
 
-//  dpm(array( 'tables' => $tables, 'default table' => $table, 'record name' => $record_name, 'priority' => $priority,
-//    'fields' => $chad_fields, 'default field' => $field, 'foreign field=>table' => $foreign_field2table,
-//    'table desc' => $table_description, 'foreign record options' => $fk_options, 'foreign table' => $foreign_table
-//    ), 'Variables');
 
   // Start of Form Proper--------------------------------------------------------------
 
@@ -1685,7 +1788,7 @@ function tripal_bulk_loader_edit_template_field_form(&$form_state = NULL) {
     '#options' => array(
       'table field' => t('Data Field: Fields which maps to a data file column'),
       'constant' => t('Constant: Field which remains Constant throughout the data file'),
-      'foreign key' => t('Foreign Key: Fields which map to a record in another table'),
+      'foreign key' => t('Record Referral: Fields which map to a record in another table'),
     ),
     '#required' => TRUE,
     '#default_value' => $field_type,
@@ -1733,6 +1836,37 @@ function tripal_bulk_loader_edit_template_field_form(&$form_state = NULL) {
     '#title' => t('Human-readable Title for Field'),
     '#default_value' => ($form_state['values']['field_title']) ? $form_state['values']['field_title'] : $template_field['title'],
   );
+  
+   // Chado Field
+  $form['edit_fields']['chado'] = array(
+    '#type' => 'fieldset',
+    '#title' => t('Chado Field/Column Details'),
+    '#description' => t('Specify the Table/Field in chado that this field maps to.'),
+  );
+
+  $form['edit_fields']['chado']['chado_table'] = array(
+    '#type' => 'select',
+    '#title' => t('Chado Table'),
+    '#options' => $tables,
+    '#default_value' => $table,
+    '#ahah' => array(
+      'path' => 'admin/tripal/tripal_bulk_loader_template/edit_field_ahah',
+      'wrapper' => 'tripal_bulk_loader_template-edit_field',
+      'effect' => 'fade'
+      ),
+  );
+
+  $form['edit_fields']['chado']['chado_field'] = array(
+    '#type' => 'select',
+    '#title' => t('Chado Field/Column'),
+    '#options' => $chado_fields,
+    '#default_value' => ($form_state['values']['chado_field']) ? $form_state['values']['chado_field'] : $template_field['field'],
+    '#ahah' => array(
+      'path' => 'admin/tripal/tripal_bulk_loader_template/edit_field_ahah',
+      'wrapper' => 'tripal_bulk_loader_template-edit_field',
+      'effect' => 'fade'
+    ),
+  );
 
   // data file column
   $form['edit_fields']['columns'] = array(
@@ -1803,53 +1937,49 @@ function tripal_bulk_loader_edit_template_field_form(&$form_state = NULL) {
     '#default_value' => ($form_state['values']['constant_validate']) ? $form_state['values']['constant_validate'] : $template_field['exposed_validate'],
   );
 
-  // Foreign Key
+  // Foreign Key / Referrer
   $form['edit_fields']['foreign_key'] = array(
     '#type' => 'fieldset',
-    '#title' => 'Foreign Key',
+    '#title' => 'Record Referral',
     '#collapsible' => TRUE,
     '#collapsed' => ($field_type == 'foreign key')? FALSE : TRUE,
   );
-
+  
+  $form['edit_fields']['foreign_key']['show_all_records'] = array(
+    '#type' => 'checkbox',
+    '#title' => 'Refer to any record',
+    '#description' => t('By default, the bulk loader will only allow referral to records in a foreign key relationship.  To allow referral to any previous record, check this box'),
+    '#default_value' => $show_all,
+    '#ahah' => array(
+      'path' => 'admin/tripal/tripal_bulk_loader_template/edit_field_ahah',
+      'wrapper' => 'tripal_bulk_loader_template-add_field',
+      'effect' => 'fade'
+    ),    
+  );
+  
   $form['edit_fields']['foreign_key']['foreign_record'] = array(
     '#type' => 'select',
     '#title' => 'Record to refer to',
-    '#descripion' => 'Select the record that this foreign key shouold refer to. The record needs to already exist and be of the correct table.',
+    '#descripion' => 'Select the record that this value should refer to. The record needs to already exist.',
     '#options' => $fk_options,
-    '#default_value' => ($form_state['values']['foreign_record']) ? $form_state['values']['foreign_record'] : $template_field['foreign key'],
-  );
-
-  // Chado Field
-  $form['edit_fields']['chado'] = array(
-    '#type' => 'fieldset',
-    '#title' => t('Chado Field/Column Details'),
-    '#description' => t('Specify the Table/Field in chado that this field maps to.'),
-  );
-
-  $form['edit_fields']['chado']['chado_table'] = array(
-    '#type' => 'select',
-    '#title' => t('Chado Table'),
-    '#options' => $tables,
-    '#default_value' => $table,
     '#ahah' => array(
       'path' => 'admin/tripal/tripal_bulk_loader_template/edit_field_ahah',
-      'wrapper' => 'tripal_bulk_loader_template-edit_field',
+      'wrapper' => 'tripal_bulk_loader_template-add_field',
       'effect' => 'fade'
-      ),
+    ),  
+    '#default_value' => ($form_state['values']['foreign_record']) ? $form_state['values']['foreign_record'] : $template_field['foreign key'],
   );
-
-  $form['edit_fields']['chado']['chado_field'] = array(
+  
+  $form['edit_fields']['foreign_key']['foreign_field'] = array(
     '#type' => 'select',
-    '#title' => t('Chado Field/Column'),
-    '#options' => $chado_fields,
-    '#default_value' => ($form_state['values']['chado_field']) ? $form_state['values']['chado_field'] : $template_field['field'],
-    '#ahah' => array(
-      'path' => 'admin/tripal/tripal_bulk_loader_template/edit_field_ahah',
-      'wrapper' => 'tripal_bulk_loader_template-edit_field',
-      'effect' => 'fade'
-    ),
+    '#title' => 'Field to refer to',
+    '#descripion' => 'Select the record that this value should refer to. The record needs to already exist.',
+    '#options' => $ref_chado_fields,
+    '#default_value' => ($form_state['values']['foreign_field']) ? $form_state['values']['foreign_field'] : $template_field['foreign field'],
   );
 
+ 
+
   $form['edit_fields']['additional'] = array(
     '#type' => 'fieldset',
     '#title' => 'Additional Options',
@@ -1857,7 +1987,7 @@ function tripal_bulk_loader_edit_template_field_form(&$form_state = NULL) {
 
   $form['edit_fields']['additional']['required'] = array(
     '#type' => 'checkbox',
-    '#title' => 'Make this file required',
+    '#title' => 'Make this field required',
     '#default_value' => $template_field['required'],
   );
 
@@ -2083,7 +2213,10 @@ function tripal_bulk_loader_edit_template_field_form_submit($form, &$form_state)
           $field['type'] = 'foreign key';
           $field['title'] = $form_state['values']['field_title'];
           $field['field'] = $form_state['values']['chado_field'];
+          $field['show_all_records'] = $form_state['values']['show_all_records'];
           $field['foreign key'] = $form_state['values']['foreign_record'];
+          $field['foreign field'] = $form_state['values']['foreign_field'];
+
           $field['required'] = $form_state['values']['required'];
       }
 

+ 87 - 20
tripal_bulk_loader/tripal_bulk_loader.loader.inc

@@ -147,8 +147,15 @@ function tripal_bulk_loader_load_data($nid, $job_id) {
   $field2column = array();
   $record2priority = array();
   $tables = array();
+  $template_array = $node->template->template_array;
 
-  foreach ($node->template->template_array as $priority => $record_array) {
+  // first build the record2priority array
+  foreach ($template_array as $priority => $record_array) {
+    $record2priority[$record_array['record_id']] = $priority;
+  }
+
+  //
+  foreach ($template_array as $priority => $record_array) {
     if (!is_array($record_array)) {
       continue;
     }
@@ -169,7 +176,6 @@ function tripal_bulk_loader_load_data($nid, $job_id) {
       $default_data[$priority]['optional'] = ($record_array['optional']) ? $record_array['optional'] : 0;
       $default_data[$priority]['select_optional'] = ($record_array['select_optional']) ? $record_array['select_optional'] : 0;
       $default_data[$priority]['record_id'] = $record_array['record_id'];
-      $record2priority[$record_array['record_id']] = $priority;
       $default_data[$priority]['required'][$field_array['field']] = $field_array['required'];
 
       $one = $default_data[$priority];
@@ -191,9 +197,31 @@ function tripal_bulk_loader_load_data($nid, $job_id) {
       }
       elseif (preg_match('/foreign key/', $field_array['type'])) {
         $default_data[$priority]['values_array'][$field_array['field']] = array();
-        $default_data[$priority]['values_array'][$field_array['field']]['foreign record'] = $field_array['foreign key'];
         $default_data[$priority]['need_further_processing'] = TRUE;
-
+        $default_data[$priority]['values_array'][$field_array['field']]['foreign record']['record'] = $field_array['foreign key'];        
+        
+        // Add in the FK / Referral table
+        $fk_priority = $record2priority[$field_array['foreign key']];
+        $fk_table = $template_array[$fk_priority]['table'];    
+        $default_data[$priority]['values_array'][$field_array['field']]['foreign record']['table'] = $fk_table;
+        
+        // Add in the FK / Referral field
+        // for backwards compatibility we need to get the FK relationship to find
+        // out what field we're joining on.  For templates created using a 
+        // previous version it was assumed that the FK field was always the field to join
+        if (!array_key_exists('foreign field', $field_array)) {
+          $tbl_description = tripal_core_get_chado_table_schema($record_array['table']);
+          foreach ($tbl_description['foreign keys'] as $key_table => $key_array) {
+            if ($key_table == $fk_table) {
+              foreach ($key_array['columns'] as $left_field => $right_field) {
+                if ($left_field == $field_array['field']) {                  
+                  $field_array['foreign field'] = $right_field;
+                }
+              }
+            }
+          }
+        }
+        $default_data[$priority]['values_array'][$field_array['field']]['foreign record']['field'] = $field_array['foreign field'];
       }
       else {
         print 'WARNING: Unsupported type: ' . $field_array['type'] . ' for ' . $table . '.' . $field_array['field'] . "!\n";
@@ -455,7 +483,7 @@ function process_data_array_for_line($priority, &$data, &$default_data, $addt) {
       //watchdog('T_bulk_loader', 'Line ' . $addt->line_num . ' Data File Added:' . print_r($values, TRUE), array(), WATCHDOG_NOTICE);
     }
 
-    $values = tripal_bulk_loader_add_foreignkey_to_values($values, $data, $addt->record2priority);
+    $values = tripal_bulk_loader_add_foreignkey_to_values($table_data, $values, $data, $addt->record2priority, $addt->nid, $priority);
 
     if (!$values) {
       //watchdog('T_bulk_loader', 'Line ' . $addt->line_num . ' FK Added:<pre>' . print_r($values, TRUE) . print_r($data[$priority], TRUE) . '</pre>', array(), WATCHDOG_NOTICE);
@@ -473,8 +501,9 @@ function process_data_array_for_line($priority, &$data, &$default_data, $addt) {
   // for an insert, check that all database required fields are present in the values array
   // we check for 'optional' in the mode for backwards compatibility. The 'optional' 
   // mode used to be a type of insert
-  if (preg_match('/insert/', $table_data['mode']) or preg_match('/optional/', $table_data['mode'])) {
-    // Check all db required fields are set
+  if (preg_match('/insert/', $table_data['mode']) or 
+      preg_match('/optional/', $table_data['mode'])) {
+    // Check all database table required fields are set
     $fields = $table_desc['fields'];
     foreach ($fields as $field => $def) {
       // a field is considered missing if it cannot be null and there is no default
@@ -500,7 +529,9 @@ function process_data_array_for_line($priority, &$data, &$default_data, $addt) {
       if (!isset($values[$field]) or (is_array($values[$field]) and count($values[$field]) == 0)){
         // check if the record is optional.  For backwards compatiblity we need to
         // check if the 'mode' is set to 'optional'
-        if ($table_data['optional'] or preg_match('/optional/', $table_data['mode']) or $table_data['select_optional'])  {
+        if ($table_data['optional'] or preg_match('/optional/', $table_data['mode']) or 
+            $table_data['select_optional'])  {
+            
           $skip_optional = 1;
           // set the values array to be empty since we all required fields are
           // optional and we can't do a select/insert so we don't want to keep
@@ -518,7 +549,7 @@ function process_data_array_for_line($priority, &$data, &$default_data, $addt) {
     }
   }
 
-  // add new values array into the data array
+  // add updated values array into the data array
   $data[$priority]['values_array'] = $values;
   
   // if there was an error already -> don't insert
@@ -647,7 +678,7 @@ function process_data_array_for_line($priority, &$data, &$default_data, $addt) {
           return $no_errors;
         }//end of if insert record
       }// end of if keeping track of records inserted
-      
+    
       // substitute the values array for the primary key if it exists
       // and is a single field
       if(array_key_exists('primary key',$table_desc)){
@@ -742,23 +773,59 @@ function tripal_bulk_loader_add_spreadsheetdata_to_values($values, $line, $field
  * record is looked up and the values array is substituted in.
  *
  */
-function tripal_bulk_loader_add_foreignkey_to_values($values, $data, $record2priority) {
-
+function tripal_bulk_loader_add_foreignkey_to_values($table_array, $values, $data, $record2priority, $nid, $priority) {
+    
   foreach ($values as $field => $value) {
     // if the field value is an array then it is an FK
     if (is_array($value)) {
-   
-      // get the name of the foreign record
-      $foreign_record   = $value['foreign record'];
       
-      // get the corresponding priority of the foreign record
+      // get the name and priority of the foreign record
+      $foreign_record   = $value['foreign record']['record'];
       $foreign_priority = $record2priority[$foreign_record];
+      $foreign_table    = $value['foreign record']['table'];
+      $foreign_field    = $value['foreign record']['field'];
       
-      // get the values of the foreign record
+      // get the values of the foreign record and substitute those for the values
       $foreign_values   = $data[$foreign_priority]['values_array'];
-      
-      // substitue the foreign values for this fields values
-      $values[$field] = $foreign_values;
+            
+      // if the field in the Referral records is in a FK relationship with
+      // this field then we can simply keep the value we have
+      $tbl_description = tripal_core_get_chado_table_schema($table_array['table']);
+      if ($foreign_field = $tbl_description['foreign keys'][$foreign_table]['columns'][$field]){
+         $values[$field] = $foreign_values;      
+      } 
+      // if the field in the Referral records is not in an FK relationship
+      // with this field then we we have to get the requested value, we must
+      // return only a single value
+      else {      
+        // if the current value of the referral records is a non-array then this
+        // is the primary key, we can use it to select the value we need.
+        $fk_description = tripal_core_get_chado_table_schema($foreign_table);
+        if (!is_array($foreign_values)) {
+          // if we have a value then use it to get the field we need
+          if ($foreign_values) {
+            $values  = array($fk_description['primary key'][0] => $foreign_values);
+            $columns = array($foreign_field);
+            $options = array('statement_name' => 'pk_' . $foreign_table);
+            $record  = tripal_core_chado_select($foreign_table, $values, $columns, $options);
+            $values[$field] = $record[0]->$foreign_field;
+          }
+          // if we don't have a value then there's nothing we can do so
+          // set this value to nothing as well
+          else {
+            unset($values[$field]);
+          }
+        }
+        // if the current value is an array and our field is not in it, then
+        // we need to select a value for our field.
+        else {
+           $values  = $foreign_values;
+           $columns = array($foreign_field);
+           $options = array('statement_name' => 'blk_' . $nid . $priority . $foreign_table);        
+           $record  = tripal_core_chado_select($foreign_table, $values, $columns, $options);
+           $values[$field] = $record[0]->$foreign_field;
+        }         
+      }
     }
   }