fixed conflicts to merge back in 6.x-0.4-dev

base/tripal_bulk_loader/tripal_bulk_loader.constants.inc

@@ -0,0 +1,548 @@
+<?php
+
+/**
+ * Inserts/Updates a tripal bulk loading job constant
+ *
+ * @param $nid
+ *   The node ID of the the tripal bulk loading job the constant is associated with
+ * @param $table
+ *   The chado table the constant is associated with
+ * @param $field
+ *   The chado field the constant is associated with
+ * @param $record_id
+ *   The index in the template array for this record
+ * @param $field_id
+ *   The index in the template array for this field
+ *
+ * NOTE: $template_array[$record_id]['table'] = $table and $template_array[$record_id]['fields'][$field_id]['field'] = $field
+ *   both are included as a means of double-checking the constant still is still in thesame place in the template array.
+ *   For example, that the template was not edited and the records moved around after the job was submitted but before it was run.
+ *
+ * @return
+ *   On success it returns the object (with primary key if inserted);
+ *   on failure it returns FALSE
+ */
+function tripal_bulk_loader_update_constant ($nid, $group_id, $table, $field, $record_id, $field_id, $value) {
+  
+  $record = array(
+    'nid'=>$nid, 
+    'group_id' => $group_id,
+    'chado_table'=>$table, 
+    'chado_field'=>$field, 
+    'record_id'=>$record_id, 
+    'field_id'=>$field_id, 
+    'value'=>$value
+  );
+  
+  // Check to see if already exists
+  $exists = db_fetch_object(db_query(
+    "SELECT constant_id FROM {tripal_bulk_loader_constants} WHERE nid=%d AND record_id=%d AND field_id=%d AND group_id=%d",
+    $record['nid'],
+    $record['record_id'],
+    $record['field_id'],
+    $record['group_id']
+  ));
+  if ($exists->constant_id) {
+    $record['constant_id'] = $exists->constant_id;
+    $status = drupal_write_record('tripal_bulk_loader_constants',$record,'constant_id');
+    if ($status) {
+      return $record;
+    } else {
+      return FALSE;
+    }
+  } else {
+
+    $status = drupal_write_record('tripal_bulk_loader_constants',$record);
+    if ($status) {
+      return $record;
+    } else {
+      return FALSE;
+    }
+  }
+}
+
+function tripal_bulk_loader_has_exposed_fields ($node) {
+
+  // exposed fields isn't set
+  if (!isset($node->exposed_fields)) {
+    return FALSE;
+  }
+  
+  // exposed fields has at least one element
+  if (sizeof($node->exposed_fields) == 1) {
+    // need to check if single element is an empty array
+    $element = reset($node->exposed_fields);
+    if ($element) {
+      return TRUE;
+    } else {
+      return FALSE;
+    }
+  } elseif (sizeof($node->exposed_fields) > 1) {
+    return TRUE;
+  } else {
+    return FALSE;
+  }
+  
+  return FALSE;
+}
+
+///////////////////////////////////////////////////////////
+// Set Constants Form (on Bulk Loader Node)
+///////////////////////////////////////////////////////////
+
+/**
+ * Set constants (exposed fields in template)
+ *
+ * @param $form_state 
+ *   The current state of the form
+ * @param $node
+ *   The node to set constants for
+ *
+ * @return
+ *   A form array to be rendered by drupal_get_form()
+ */
+function tripal_bulk_loader_set_constants_form ($form_state, $node) {
+  $form = array();
+  
+  $form['nid'] = array(
+    '#type' => 'hidden',
+    '#value' => $node->nid
+  );
+  
+  if (!tripal_bulk_loader_has_exposed_fields($node)) {
+    return $form;
+  }
+  
+  $form['exposed_array'] = array(
+    '#type' => 'hidden',
+    '#value' => serialize($node->exposed_fields),
+  );
+  
+  $form['exposed_fields'] = array(
+    '#type' => 'fieldset',
+    '#title' => t('Constant Values'),
+    '#collapsible' => TRUE,
+    '#collapsed' => ($node->template_id) ? FALSE : TRUE,
+    '#prefix' => '<div id="set-constants">',
+    '#suffix' => '</div>',
+  );
+
+  // Display table of already added constant sets with the ability to re-arrange and delete
+  if (sizeof($node->constants) > 0) {
+    $form['exposed_fields']['explanation-1'] = array(
+      '#type' => 'item',
+      '#value' => t('You have already added constants to this bulk loading job. Each '
+        .'row in the following table represents a set of constants. Each set will be used '
+        .'to load your spreadsheet with the specified template resulting in the each record '
+        .'in the template to be loaded x number of times where there are x sets of '
+        .'constants (rows in the following table).')
+    );
+    
+    $form['exposed_fields']['existing'] = array(
+      '#tree' => TRUE,
+    );
+    
+    foreach ($node->constants as $set) {
+
+      foreach ($set as $record) {
+        foreach ($record as $field) {
+          $index = $field['record_id'].'-'.$field['field_id'];
+          $group = $field['group_id'];
+          $form['exposed_fields']['existing'][$group][$index] = array(
+            '#type' => 'markup',
+            '#value' => $field['value'],
+          );
+        }
+      }
+
+      $form['exposed_fields']['existing'][$group]['delete'] = array(
+        '#type' => 'markup',
+        '#value' => l('Edit', 'node/'. $node->nid . '/constants/'.$group.'/edit') .'<br />'.
+          l('Delete', 'node/'. $node->nid . '/constants/'.$group.'/delete'),
+      );
+                  
+    }
+  }
+  
+  $form['exposed_fields']['new'] = array(
+    '#type' => 'fieldset',
+    '#title' => t('New set of Constants'),
+  );
+    
+  $form['exposed_fields']['new']['explanation-2'] = array(
+    '#type' => 'item',
+    '#value' => t('The following fields are constants in the selected template that you need to set values for.')
+  );
+  
+  // Add textifelds for exposed fields of the current template
+  $exposed_fields = FALSE;
+  $indexes = array();
+  if (tripal_bulk_loader_has_exposed_fields($node)) {
+    foreach ($node->exposed_fields as $exposed_index) {
+      
+      $record_id = $exposed_index['record_id'];
+      $field_id = $exposed_index['field_id'];
+      $field = $node->template->template_array[$record_id]['fields'][$field_id];
+      
+      if ($field['exposed']) {
+        $exposed_fields = TRUE;
+        $indexes[$record_id][] = $field_id;
+        
+        switch($field['type']) {
+          case 'table field':
+            $form['exposed_fields']['new'][$record_id.'-'.$field_id] = array(
+              '#type' => 'textfield',
+              '#title' => t($field['title']),
+              '#description' => t($field['exposed_description']),
+              '#default_value' => (isset($node->constants[$record_id][$field_id]['value'])) ? $node->constants[$record_id][$field_id]['value'] : $field['constant value'],
+            );
+          break;
+          case 'constant':
+            $form['exposed_fields']['new'][$record_id.'-'.$field_id] = array(
+              '#type' => 'textfield',
+              '#title' => t($field['title']),
+              '#description' => t('Enter the case-sensitive value of this constant for your spreadsheet'),
+              '#default_value' => (isset($node->constants[$record_id][$field_id]['value'])) ? $node->constants[$record_id][$field_id]['value'] : $field['constant value'],
+            );
+          break;
+        }
+        
+        $form['exposed_fields']['new'][$record_id.'-'.$field_id.'-table'] = array(
+          '#type' => 'hidden',
+          '#value' => $node->template->template_array[$record_id]['table'],
+        );
+        $form['exposed_fields']['new'][$record_id.'-'.$field_id.'-field'] = array(
+          '#type' => 'hidden',
+          '#value' => $field['field'],
+        );
+        $form['exposed_fields']['new'][$record_id.'-'.$field_id.'-type'] = array(
+          '#type' => 'hidden',
+          '#value' => $field['type'],
+        );
+
+      }
+    }
+  }
+  $form['template'] = array(
+    '#type' => 'hidden',
+    '#value' => serialize($node->template->template_array)
+  );
+  
+  $form['exposed_fields']['new']['indexes'] = array(
+    '#type' => 'hidden',
+    '#value' => serialize($indexes),
+  );
+  
+  if (!$exposed_fields) {
+    $form['exposed_fields']['new']['explanation'] = array(
+      '#type' => 'item',
+      '#value' => t('There are no exposed fields for this template.')
+    );    
+  }
+  
+  $form['exposed_fields']['new']['submit-2'] = array(
+    '#type' => 'submit',
+    '#name' => 'add_constant',
+    '#value' => t('Add Constant Set')
+  );
+  
+  return $form;
+}
+
+/**
+ * Validate that the values entered exist in the database
+ * if indicated in hte template array
+ */
+function tripal_bulk_loader_set_constants_form_validate ($form, $form_state) {
+
+  $template = unserialize($form_state['values']['template']);
+  $indexes = unserialize($form_state['values']['indexes']);
+
+  $op = $form_state['values'][ $form_state['clicked_button']['#name'] ];
+  if (strcmp('Add Constant Set',$op) == 0) {
+      foreach ($indexes as $record_id => $array) {
+        foreach($array as $field_id) {
+          if ($template[$record_id]['fields'][$field_id]['exposed_validate']) {
+            $result = db_fetch_object(db_query(
+              "SELECT 1 as valid FROM %s WHERE %s='%s'",
+              $template[$record_id]['table'],
+              $template[$record_id]['fields'][$field_id]['field'],
+              $form_state['values'][$record_id.'-'.$field_id]
+            ));
+    
+            if (!$result->valid) {
+              $msg = 'A '.$form['exposed_fields']['new'][$record_id.'-'.$field_id]['#title'].' of "'.$form['exposed_fields']['new'][$record_id.'-'.$field_id]['#value'].'" must already exist!';
+              form_set_error($record_id.'-'.$field_id, $msg);
+            } else {
+              drupal_set_message('Confirmed a '.$form['exposed_fields']['new'][$record_id.'-'.$field_id]['#title'].' of "'.$form['exposed_fields']['new'][$record_id.'-'.$field_id]['#value'].'" already exists.');
+            }
+          }
+        }
+      }
+  }
+  
+}
+
+/**
+ * Insert/update the constants associated with this node
+ */
+function tripal_bulk_loader_set_constants_form_submit ($form, $form_state) {
+
+	// Insert/Update constants
+  $template = unserialize($form_state['values']['template']);
+  $indexes = unserialize($form_state['values']['indexes']);
+
+  $op = $form_state['values'][ $form_state['clicked_button']['#name'] ];
+  if (strcmp('Add Constant Set',$op) == 0) {
+      $max_group = db_fetch_object(db_query("SELECT max(group_id) as value FROM tripal_bulk_loader_constants WHERE nid=%d",$form_state['values']['nid']));
+      foreach ($indexes as $record_id => $array) {
+        foreach($array as $field_id) {	
+          tripal_bulk_loader_update_constant(
+            $form_state['values']['nid'], 
+            $max_group->value+1,
+            $form_state['values'][$record_id.'-'.$field_id.'-table'], 
+            $form_state['values'][$record_id.'-'.$field_id.'-field'],
+            $record_id,
+            $field_id,
+            $form_state['values'][$record_id.'-'.$field_id]
+          );
+        }
+      }
+  }
+  
+}
+
+function theme_tripal_bulk_loader_set_constants_form ($form) {
+  $output = '';
+  
+  $exposed_fields = unserialize($form['exposed_array']['#value']);
+  // need to put in the context of a node so we can use the has_exposed_fields function
+  if ($exposed_fields) {
+    $node->exposed_fields = $exposed_fields;
+  } else {
+    $node->exposed_fields = array();
+  }
+  
+  // Add draggable table for constant sets
+  if (tripal_bulk_loader_has_exposed_fields($node)) {
+    $i=1;
+    foreach (element_children($form['exposed_fields']['existing']) as $key) {
+      $element = &$form['exposed_fields']['existing'][$key];
+      $element['group']['#attributes']['class'] = 'weight-group';
+  
+      $row = array();
+      foreach ($exposed_fields as $exposed) {
+        if ($i==1) {
+          $header[] = $exposed['title'];
+        }
+        $k = $exposed['record_id'].'-'.$exposed['field_id'];
+        $row[] = drupal_render($element[$k]);
+      }
+      $row[] = drupal_render($element['delete']);
+      $row[] = drupal_render($element['group']) . drupal_render($element['id']);
+      $rows[] = array('data' => $row, 'class' => 'draggable');
+      $i++;
+    }
+    //drupal_add_tabledrag('mytable', 'order', 'sibling', 'weight-group');
+    $form['exposed_fields']['existing'] = array(
+      '#type' => 'markup',
+      '#value' => theme('table', $header, $rows, array('id' => 'mytable')) . '<br />'
+    );
+  }
+    
+  $output .= drupal_render($form);
+  return $output;
+}
+
+///////////////////////////////////////////////////////////
+// Set Constants Form (on Bulk Loader Node)
+///////////////////////////////////////////////////////////
+
+/**
+ * Edit a constant set (exposed fields in template)
+ *
+ * @param $form_state 
+ *   The current state of the form
+ * @param $node
+ *   The node to set constants for
+ * @param $group_id
+ *   The constant set to edit
+ *
+ * @return
+ *   A form array to be rendered by drupal_get_form()
+ */
+function tripal_bulk_loader_edit_constant_set_form ($form_state, $node, $group_id) {
+  $form = array();
+
+  $form['#redirect'] = 'node/'.$node->nid;
+  
+  $form['nid'] = array(
+    '#type' => 'hidden',
+    '#value' => $node->nid,
+  );
+
+  $form['group_id'] = array(
+    '#type' => 'hidden',
+    '#value' => $group_id,
+  );
+
+    
+  $form['explanation'] = array(
+    '#type' => 'item',
+    '#value' => t('The following fields are constants in the selected template that you need to set values for.')
+  );
+  
+  // Add textifelds for exposed fields of the current template
+  $exposed_fields = FALSE;
+  $indexes = array();
+  if (tripal_bulk_loader_has_exposed_fields($node)) {
+    foreach ($node->exposed_fields as $exposed_index) {
+      
+      $record_id = $exposed_index['record_id'];
+      $field_id = $exposed_index['field_id'];
+      $field = $node->template->template_array[$record_id]['fields'][$field_id];
+
+      if ($field['exposed']) {
+        $exposed_fields = TRUE;
+        $indexes[$record_id][] = $field_id;
+        
+        switch($field['type']) {
+          case 'table field':
+            $form[$record_id.'-'.$field_id] = array(
+              '#type' => 'textfield',
+              '#title' => t($field['title']),
+              '#description' => t($field['exposed_description']),
+              '#default_value' => (isset($node->constants[$group_id][$record_id][$field_id]['value'])) ? $node->constants[$group_id][$record_id][$field_id]['value'] : $field['constant value'],
+            );
+          break;
+          case 'constant':
+            $form[$record_id.'-'.$field_id] = array(
+              '#type' => 'textfield',
+              '#title' => t($field['title']),
+              '#description' => t('Enter the case-sensitive value of this constant for your spreadsheet'),
+              '#default_value' => (isset($node->constants[$group_id][$record_id][$field_id]['value'])) ? $node->constants[$group_id][$record_id][$field_id]['value'] : $field['constant value'],
+            );
+          break;
+        }
+        
+        $form[$record_id.'-'.$field_id.'-table'] = array(
+          '#type' => 'hidden',
+          '#value' => $record['table'],
+        );
+        $form[$record_id.'-'.$field_id.'-field'] = array(
+          '#type' => 'hidden',
+          '#value' => $field['field'],
+        );
+        $form[$record_id.'-'.$field_id.'-type'] = array(
+          '#type' => 'hidden',
+          '#value' => $field['type'],
+        );
+      }
+
+    }
+  }
+  $form['template'] = array(
+    '#type' => 'hidden',
+    '#value' => serialize($node->template->template_array)
+  );
+  
+  $form['indexes'] = array(
+    '#type' => 'hidden',
+    '#value' => serialize($indexes),
+  );
+  
+  $form['save'] = array(
+    '#type' => 'submit',
+    '#value' => 'Save',
+  );
+  
+  $form['cancel'] = array(
+    '#type' => 'submit',
+    '#value' => 'Cancel',
+  );
+    
+  return $form;
+}
+
+/**
+ * Edit constants in the current constant set
+ */
+function tripal_bulk_loader_edit_constant_set_form_submit ($form, $form_state) {
+  
+  // Update constants
+  $template = unserialize($form_state['values']['template']);
+  $indexes = unserialize($form_state['values']['indexes']);
+  
+  $op = $form_state['values'][ $form_state['clicked_button']['#name'] ];
+  if (strcmp('Save',$op) == 0) {
+
+    foreach ($indexes as $record_id => $array) {
+      foreach($array as $field_id) {	
+        tripal_bulk_loader_update_constant(
+          $form_state['values']['nid'], 
+          $form_state['values']['group_id'],
+          $form_state['values'][$record_id.'-'.$field_id.'-table'], 
+          $form_state['values'][$record_id.'-'.$field_id.'-field'],
+          $record_id,
+          $field_id,
+          $form_state['values'][$record_id.'-'.$field_id]
+        );
+      }
+    }
+    drupal_set_message('The constant set was successfully updated.');
+      
+  }
+  
+}
+
+/**
+ * Delete a constant set (exposed fields in template)
+ *
+ * @param $form_state 
+ *   The current state of the form
+ * @param $node
+ *   The node to set constants for
+ * @param $group_id
+ *   The constant set to delete
+ *
+ * @return
+ *   A form array to be rendered by drupal_get_form()
+ */
+function tripal_bulk_loader_delete_constant_set_form ($form_state, $node, $group_id) {
+  $form = array();
+
+  $form['#redirect'] = 'node/'.$node->nid;
+	
+	$form['nid'] = array(
+		'#type' => 'value',
+		'#value' => $node->nid,
+	);
+
+  $form['group_id'] = array(
+    '#type' => 'hidden',
+    '#value' => $group_id,
+  );
+   
+	return confirm_form($form,
+    	t('Are you sure you want to delete this constant set?'),
+    	'node/'.$node->nid,
+    	t('This action cannot be undone.'),
+    	t('Delete'),
+    	t('Cancel')
+  );
+  
+}
+
+/**
+ * Delete the current constant set
+ */
+function tripal_bulk_loader_delete_constant_set_form_submit ($form, $form_state) {
+
+  $group_id = $form_state['values']['group_id'];
+  $nid = $form_state['values']['nid'];
+  if ($nid && $form_state['values']['confirm']) {
+    db_query("DELETE FROM {tripal_bulk_loader_constants} WHERE nid=%d AND group_id=%d",$nid, $group_id);
+    drupal_set_message('Constant set successfully deleted.');
+  }
+        
+}

base/tripal_bulk_loader/tripal_bulk_loader.install

@@ -114,26 +114,32 @@ function tripal_bulk_loader_schema() {
         'type' => 'int',
         'unsigned' => TRUE,
         'not null' => TRUE,
-			),
-			'chado_table' => array(
-			  'type' => 'varchar',
-			  'not null' => TRUE,
-			),
-			'chado_field' => array(
-			  'type' => 'varchar',
-			  'not null' => TRUE,
-			),
-			'record_id' => array(
-			  'type' => 'int',
-			  'not null' => TRUE
-			),
-			'field_id' => array(
-			  'type' => 'int',
-			  'not null' => TRUE
-			),
-			'value' => array(
-			  'type' => 'text',
-			),
+      ),
+      'group_id' => array(
+        'type' => 'int',
+        'unsigned' => TRUE,
+        'not null' => TRUE,
+        'default' => 0
+      ),
+      'chado_table' => array(
+        'type' => 'varchar',
+        'not null' => TRUE,
+      ),
+      'chado_field' => array(
+        'type' => 'varchar',
+        'not null' => TRUE,
+      ),
+      'record_id' => array(
+        'type' => 'int',
+        'not null' => TRUE
+      ),
+      'field_id' => array(
+        'type' => 'int',
+        'not null' => TRUE
+      ),
+      'value' => array(
+        'type' => 'text',
+      ),
 	  ),
 	  'primary key' => array('constant_id'),
 	);
@@ -154,4 +160,28 @@ function tripal_bulk_loader_update_6150 () {
   
   return $ret;
   
+}
+
+/**
+ * Update schema for version 6.x-0.3.1b-1.5
+ * - Add the tripal_bulk_loader_constants.group_id column 
+ *   to allow multiple sets of constants per job
+ */
+function tripal_bulk_loader_update_6151 () {
+  $ret = array();
+  
+  $schema = tripal_bulk_loader_schema();
+  db_add_field(
+    $ret, 
+    'tripal_bulk_loader_constants', 
+    'group_id', 
+    array(
+        'type' => 'int',
+        'unsigned' => TRUE,
+        'not null' => TRUE,
+        'default' => 0
+      )
+    );
+
+  return $ret;  
 }

base/tripal_bulk_loader/tripal_bulk_loader.loader.inc

@@ -110,23 +110,13 @@ function tripal_bulk_loader_load_data($nid) {
   
   // set the status of the job (in the node not the tripal jobs)
   db_query("UPDATE tripal_bulk_loader SET job_status='%s' WHERE nid=%d",'Loading...', $nid);
-  
-  print "Memory Usage (Start): ".number_format((memory_get_usage() * 0.000000953674316), 5, '.', ',') . " Mb\n";
+
   
   $node = node_load($nid);
   print "Template: ".$node->template->name." (".$node->template_id.")\n";
-  print "File: ".$node->file."\n";
-  
-  if ($node->constants) {
-    print "Constants:\n";
-    foreach ($node->constants as $record_id => $record) {
-      foreach ($record as $field_id => $field) {
-        print "\t- ".$field['chado_table'].'.'.$field['chado_field'].' = '.$field['value']."\n";
-      }
-    }
-  }
-  
-  print "\nMemory Usage (After Node Load): ".number_format((memory_get_usage() * 0.000000953674316), 5, '.', ',') . " Mb\n";
+
+  $total_lines = trim(`wc --lines < $node->file`);
+  print "File: ".$node->file." (".$total_lines." lines)\n";
   
   // Prep Work ==================================================================================
   $loaded_without_errors = TRUE;
@@ -180,86 +170,101 @@ function tripal_bulk_loader_load_data($nid) {
     //watchdog('T_bulk_loader','2)'.$record_array['record_id'].':<pre>'.print_r($default_data[$priority], TRUE).'</pre>', array(), WATCHDOG_NOTICE);
   } //end of foreach record
 
-  // Add constants
-  if ($node->constants) {
-    foreach ($node->constants as $priority => $record) {
-      foreach ($record as $field_id => $field) {
-        if ($default_data[$priority]['table'] == $field['chado_table']) {
-          if (isset($default_data[$priority]['values_array'][$field['chado_field']])) {
-            if (isset($field2column[$priority][$field['chado_field']])) {
-              $field2column[$priority][$field['chado_field']] = $field['value'];
+  ///////////////////////////////////////////////
+  // For each set of constants
+  ///////////////////////////////////////////////
+  $original_default_data = $default_data;
+  $group_index = 0;
+  $total_num_groups = sizeof($node->constants);
+  foreach ($node->constants as $group_id => $set) {
+    // revert default data array for next set of constants
+    $default_data = $original_default_data;
+    $group_index++;
+    
+    // Add constants
+    if (!empty($set)) {
+      print "Constants:\n";
+      foreach ($set as $priority => $record) {
+        foreach ($record as $field_id => $field) {
+        
+          print "\t- ".$field['chado_table'].'.'.$field['chado_field'].' = '.$field['value']."\n";
+          
+          if ($default_data[$priority]['table'] == $field['chado_table']) {
+            if (isset($default_data[$priority]['values_array'][$field['chado_field']])) {
+              if (isset($field2column[$priority][$field['chado_field']])) {
+                $field2column[$priority][$field['chado_field']] = $field['value'];
+              } else {
+                $default_data[$priority]['values_array'][$field['chado_field']] = $field['value'];
+              }
             } else {
-              $default_data[$priority]['values_array'][$field['chado_field']] = $field['value'];
+              print "ERROR: Template has changed after constants were assigned!\n";
+              watchdog('T_bulk_loader','Template has changed after constants were assigned', array(), WATCHDOG_NOTICE);
+              exit(1);
             }
           } else {
             print "ERROR: Template has changed after constants were assigned!\n";
             watchdog('T_bulk_loader','Template has changed after constants were assigned', array(), WATCHDOG_NOTICE);
             exit(1);
           }
-        } else {
-          print "ERROR: Template has changed after constants were assigned!\n";
-          watchdog('T_bulk_loader','Template has changed after constants were assigned', array(), WATCHDOG_NOTICE);
-          exit(1);
         }
       }
     }
-  }
-  
-  print "Memory Usage (end of prep work): ".number_format((memory_get_usage() * 0.000000953674316), 5, '.', ',') . " Mb\n";
-  
-  //print "Default Data:".print_r($default_data,TRUE)."\n";
-  //watchdog('T_bulk_loader','Default Data:<pre>'.print_r($default_data, TRUE).'</pre>', array(), WATCHDOG_NOTICE);
-  
-  //print "\nDefault Values Array: ".print_r($default_data, TRUE)."\n";
-  //print "\nField to Column Mapping: ".print_r($field2column, TRUE)."\n";
-  
-  // Parse File adding records as we go ========================================================
-  $file_handle = fopen($node->file, 'r');
-  if (preg_match('/(t|true|1)/', $node->file_has_header)) { fgets($file_handle, 4096); }
-  $num_records = 0;
-  $num_lines = 0;
-  $num_errors = 0;
-  $total_lines = trim(`wc --lines < $node->file`);
-  $interval = intval($total_lines * 0.01);
-  if($interval == 0){ $interval = 1; }
-  while (!feof($file_handle)) {
-
-    // Clear variables
-    // Was added to fix memory leak
-    unset($line);                     unset($raw_line);
-    unset($data);                     unset($data_keys);
-    unset($priority);                 unset($sql);
-    unset($result);                   
     
-    $raw_line = fgets($file_handle, 4096);
-    $raw_line = trim($raw_line);
-    if (empty($raw_line)) { continue; } // skips blank lines
-    $line = explode("\t", $raw_line);
-    $num_lines++;
+    //print "Default Data:".print_r($default_data,TRUE)."\n";
+    //watchdog('T_bulk_loader','Default Data:<pre>'.print_r($default_data, TRUE).'</pre>', array(), WATCHDOG_NOTICE);
     
-    // update the job status every 1% features
-    if($node->job_id and $num_lines % $interval == 0){
-      print "Updating progress of ".$node->job_id." to ".($num_lines/$interval) ."%\n";
-       tripal_job_set_progress($node->job_id,($num_lines/$interval));
-    }    
-
-    $data = $default_data;
-
-    $data_keys = array_keys($data); 
-    foreach ($data_keys as $priority) {
-      $status = process_data_array_for_line($priority, $data, $default_data, $field2column, $record2priority, $line, $nid);
-      if (!$status ) { $loaded_without_errors = FALSE; }
-    } // end of foreach table in default data array
-
-    // determine memory increase
-    $line_mem_increase = memory_get_usage() - $memory;
-    if ($num_lines != 1) { 
-      $increased_mem = $increased_mem + $line_mem_increase;
-    }
-    $memory = memory_get_usage();
+    //print "\nDefault Values Array: ".print_r($default_data, TRUE)."\n";
+    //print "\nField to Column Mapping: ".print_r($field2column, TRUE)."\n";
+    
+    // Parse File adding records as we go ========================================================
+    $file_handle = fopen($node->file, 'r');
+    if (preg_match('/(t|true|1)/', $node->file_has_header)) { fgets($file_handle, 4096); }
+    $num_records = 0;
+    $num_lines = 0;
+    $num_errors = 0;
+    $interval = intval($total_lines * 0.10);
+    if($interval == 0){ $interval = 1; }
+    while (!feof($file_handle)) {
+  
+      // Clear variables
+      // Was added to fix memory leak
+      unset($line);                     unset($raw_line);
+      unset($data);                     unset($data_keys);
+      unset($priority);                 unset($sql);
+      unset($result);                   
+      
+      $raw_line = fgets($file_handle, 4096);
+      $raw_line = trim($raw_line);
+      if (empty($raw_line)) { continue; } // skips blank lines
+      $line = explode("\t", $raw_line);
+      $num_lines++;
+      
+      // update the job status every 10% of lines processed for the current group
+      if($node->job_id and $num_lines % $interval == 0){
+        // percentage of lines processed for the current group
+        $group_progress = round(($num_lines/$total_lines)*100);
+        // percentage of lines processed for all groups
+        $job_progress = round(($group_index / $total_num_groups) * $group_progress);
+        
+          print "\nProgress Update:\n"
+            ."\t- ".$num_lines." lines have been processed for the current constant set.\n"
+            ."\t- ".$group_progress."% of the lines in the file have been processed for the current constant set.\n"
+            ."\t- ".$job_progress."% of the current job has been completed.\n";
+            
+        tripal_job_set_progress($node->job_id,$job_progress);
+      }    
+  
+      $data = $default_data;
+  
+      $data_keys = array_keys($data); 
+      foreach ($data_keys as $priority) {
+        $status = process_data_array_for_line($priority, $data, $default_data, $field2column, $record2priority, $line, $nid, $num_lines);
+        if (!$status ) { $loaded_without_errors = FALSE; }
+      } // end of foreach table in default data array
+      
+    } //end of foreach line of file
+  } //end of foreach constant set
     
-  } //end of foreach line of file
-  
   // check that data was inserted and update job_status
   $sql = 'SELECT count(*) as num_tables FROM {tripal_bulk_loader_inserted} WHERE nid=%d GROUP BY nid';
   $result = db_fetch_object(db_query($sql, $nid));
@@ -271,11 +276,6 @@ function tripal_bulk_loader_load_data($nid) {
   // set the status of the job (in the node not the tripal jobs)
   if ($loaded_without_errors) { $status = 'Loading Completed Successfully'; } else { $status = 'Errors Encountered'; }
   db_query("UPDATE tripal_bulk_loader SET job_status='%s' WHERE nid=%d",$status, $nid);
-  
-  $avg_line_increase = ( $increased_mem / $num_lines) * 0.0078125;
-  print "Average Increase in Memory per Line: ".number_format($avg_line_increase, 5, '.', ',') . " Kb\n";
-  print "Peak Memory Usage: ".number_format((memory_get_peak_usage() * 0.000000953674316), 5, '.', ',') . " Mb\n";
-  print "End Memory Usage: ".number_format((memory_get_usage() * 0.000000953674316), 5, '.', ',') . " Mb\n";
 
 }
 
@@ -283,7 +283,7 @@ function tripal_bulk_loader_load_data($nid) {
  * 
  *
  */
-function process_data_array_for_line ($priority, &$data, &$default_data, $field2column, $record2priority, $line, $nid) {
+function process_data_array_for_line ($priority, &$data, &$default_data, $field2column, $record2priority, $line, $nid, $line_num) {
   $table_data = $data[$priority];
 
   $no_errors = TRUE;
@@ -297,21 +297,21 @@ function process_data_array_for_line ($priority, &$data, &$default_data, $field2
   if ($table_data['need_further_processing']) {
     $values = tripal_bulk_loader_add_spreadsheetdata_to_values ($values, $line, $field2column[$priority]);
     if (!$values) {
-      watchdog('T_bulk_loader','Spreadsheet Added:'.print_r($values, TRUE), array(), WATCHDOG_NOTICE);
+      watchdog('T_bulk_loader','Line '.$line_num.' Spreadsheet Added:'.print_r($values, TRUE), array(), WATCHDOG_NOTICE);
     }
     
     $values = tripal_bulk_loader_add_foreignkey_to_values($values, $data, $record2priority);
     if (!$values) {
-      watchdog('T_bulk_loader','FK Added:<pre>'.print_r($values, TRUE).print_r($data[$priority],TRUE).'</pre>', array(), WATCHDOG_NOTICE);
+      watchdog('T_bulk_loader','Line '.$line_num.' FK Added:<pre>'.print_r($values, TRUE).print_r($data[$priority],TRUE).'</pre>', array(), WATCHDOG_NOTICE);
     }
   }
   $values = tripal_bulk_loader_regex_tranform_values($values, $table_data, $line);
   if (!$values) {
-    watchdog('T_bulk_loader','Regex:<pre>'.print_r($values, TRUE).print_r($table_data, TRUE).'</pre>'.'</pre>', array(), WATCHDOG_NOTICE);
+    watchdog('T_bulk_loader','Line '.$line_num.' Regex:<pre>'.print_r($values, TRUE).print_r($table_data, TRUE).'</pre>'.'</pre>', array(), WATCHDOG_NOTICE);
   }
 
   if (!$values) {
-    $msg = $table_data['record_id'].' ('.$table_data['mode'].') Aborted due to error in previous record. Values of current record:'.print_r($table_data['values_array'],TRUE);
+    $msg = 'Line '.$line_num.' '.$table_data['record_id'].' ('.$table_data['mode'].') Aborted due to error in previous record. Values of current record:'.print_r($table_data['values_array'],TRUE);
     watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING); 
     print "ERROR: ".$msg."\n";
     $data[$priority]['error'] = TRUE;
@@ -326,7 +326,7 @@ function process_data_array_for_line ($priority, &$data, &$default_data, $field2
       // a field is considered missing if it cannot be null and there is no default
       // value for it or it is of type 'serial'
       if($def['not null'] == 1 and !array_key_exists($field,$insert_values) and !isset($def['default']) and strcmp($def['type'],serial)!=0){
-         $msg = $table_data['record_id'].' ('.$table_data['mode'].') Missing Database Required Value: '.$table.'.'.$field;
+         $msg = 'Line '.$line_num.' '.$table_data['record_id'].' ('.$table_data['mode'].') Missing Database Required Value: '.$table.'.'.$field;
          watchdog('T_bulk_loader', $msg, array(), WATCHDOG_NOTICE); 
          $data[$priority]['error'] = TRUE;
       }
@@ -337,7 +337,7 @@ function process_data_array_for_line ($priority, &$data, &$default_data, $field2
   foreach ($table_data['required'] as $field => $required) {
     if ($required) {
       if (!isset($values[$field])) {
-        $msg = $table_data['record_id'].' ('.$table_data['mode'].') Missing Template Required Value: '.$table.'.'.$field;
+        $msg = 'Line '.$line_num.' '.$table_data['record_id'].' ('.$table_data['mode'].') Missing Template Required Value: '.$table.'.'.$field;
         watchdog('T_bulk_loader', $msg, array(), WATCHDOG_NOTICE); 
         $data[$priority]['error'] = TRUE;  
       }
@@ -381,7 +381,7 @@ function process_data_array_for_line ($priority, &$data, &$default_data, $field2
     //watchdog('T_bulk_loader',$header.': Inserting:'.print_r($values, TRUE), array(), WATCHDOG_NOTICE);
     $record = tripal_core_chado_insert($table, $values);
     if (!$record) {
-      $msg = $table_data['record_id'].' ('.$table_data['mode'].') Unable to insert record into '.$table.' where values:'.print_r($values,TRUE);
+      $msg = 'Line '.$line_num.' '.$table_data['record_id'].' ('.$table_data['mode'].') Unable to insert record into '.$table.' where values:'.print_r($values,TRUE);
       watchdog('T_bulk_loader', $msg, array(), WATCHDOG_ERROR); 
       print "ERROR: ".$msg."\n";  
       $data[$priority]['error'] = TRUE;
@@ -424,7 +424,7 @@ function process_data_array_for_line ($priority, &$data, &$default_data, $field2
     $exists = tripal_core_chado_select($table, array_keys($table_desc['fields']), $values, array('has_record'=>TRUE));
     if (!$exists) {
       // No record on select
-      $msg = $table_data['record_id'].' ('.$table_data['mode'].') No Matching record in '.$table.' where values:'.print_r($values,TRUE);
+      $msg = 'Line '.$line_num.' '.$table_data['record_id'].' ('.$table_data['mode'].') No Matching record in '.$table.' where values:'.print_r($values,TRUE);
       watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING); 
       $data[$priority]['error'] = TRUE;  
     }

base/tripal_bulk_loader/tripal_bulk_loader.module

@@ -1,6 +1,7 @@
 <?php
 
 include('tripal_bulk_loader.loader.inc');
+include('tripal_bulk_loader.constants.inc');
 
 /**
  * Implements hook_init
@@ -25,6 +26,25 @@ function tripal_bulk_loader_menu() {
      'access arguments' => array('access tripal_bulk_loader'),
      'type' => MENU_NORMAL_ITEM,
    );
+   // Bulk Loading Job Node
+   $items['node/%node/constants/%/edit'] = array(
+      'title' => 'Edit Constant Set',
+      'description' => 'Edit a group of constants associated with the current bulk loader',
+    	'page callback' => 'drupal_get_form',
+    	'page arguments' => array('tripal_bulk_loader_edit_constant_set_form',1,3),
+      'access arguments' => array('administer site configuration'),
+      'type' => MENU_CALLBACK,   
+		  'file' => 'tripal_bulk_loader.constants.inc',
+   );
+   $items['node/%node/constants/%/delete'] = array(
+      'title' => 'Delete Constant Set',
+      'description' => 'Delete a group of constants associated with the current bulk loader',
+    	'page callback' => 'drupal_get_form',
+    	'page arguments' => array('tripal_bulk_loader_delete_constant_set_form',1,3),
+      'access arguments' => array('administer site configuration'),
+      'type' => MENU_CALLBACK,   
+		  'file' => 'tripal_bulk_loader.constants.inc',
+   );
    // Admin page to create the template
    $items['admin/tripal/tripal_bulk_loader_template'] = array(
       'title' => 'Bulk Loader Template',
@@ -110,11 +130,6 @@ function tripal_bulk_loader_menu() {
 		  'file' => 'tripal_bulk_loader.admin.inc',  
   );	
 	// AHAH ---------
-	$items['node/add/tripal-bulk-loader/ahah'] = array(
-      'page callback' => 'tripal_bulk_loader_create_node_ahah',
-      'access arguments' => array('create tripal_bulk_loader'),
-      'type' => MENU_CALLBACK,   
-	);
 	$items['admin/tripal/tripal_bulk_loader_template/add_field_ahah'] = array(
       'page callback' => 'tripal_bulk_loader_add_field_ahah',
       'access arguments' => array('administer site configuration'),
@@ -131,11 +146,15 @@ function tripal_bulk_loader_menu() {
   return $items;
 }
 
+
 /**
  * Implements hook_theme
  */
 function tripal_bulk_loader_theme() {
   return array(
+    'tripal_bulk_loader_set_constants_form' => array(
+      'arguments' => array('form' => NULL),
+    ),
     'tripal_bulk_loader_template' => array(
     	'arguments'=> array('template_id' => NULL),
     	'template' => 'tripal_bulk_loader_template'
@@ -293,13 +312,6 @@ function tripal_bulk_loader_form ($node, $form_state){
     '#weight'        => -9,
     '#required'      => TRUE,
     '#default_value' => $node->template_id,
-    '#ahah' => array(
-      'path' => 'node/add/tripal-bulk-loader/ahah',
-      'wrapper' => 'set-constants',
-      'event' => 'change',
-      'method' => 'replace',
-      'effect' => 'fade',
-    ),
   );
   
   $form['loader']['file']= array(
@@ -321,43 +333,6 @@ function tripal_bulk_loader_form ($node, $form_state){
   return $form;
 }
 
-/**
- * AHAH Function: Replace $form['constants'] in tripal_bulk_loader_form
- *
- * @return
- *  JSON Data printed to the screen
- */
-function tripal_bulk_loader_create_node_ahah () {
-
-  // The form is generated in an include file which we need to include manually.
-  include_once 'modules/node/node.pages.inc';
-  
-  $form_state = array('storage' => NULL, 'submitted' => FALSE);
-  $form_build_id = $_POST['form_build_id'];
-  $form = form_get_cache($form_build_id, $form_state);
-  $args = $form['#parameters'];
-  $form_id = array_shift($args);
-  $form_state['post'] = $form['#post'] = $_POST;
-  
-  // Enable the submit/validate handlers to determine whether AHAH-submittted.
-  $form_state['ahah_submission'] = TRUE;
-  
-  $form['#programmed'] = $form['#redirect'] = FALSE;
-  drupal_process_form($form_id, $form, $form_state);
-  $form = drupal_rebuild_form($form_id, $form_state, $args, $form_build_id);
-  
-  $form_element = $form['exposed_fields'];
-  // Remove the wrapper so we don't double it up.
-  unset($form_element['#prefix'], $form_element['#suffix']);
-
-  $output = theme('status_messages');
-  $output .= drupal_render($form_element);
-
-  // Final rendering callback.
-  print drupal_json(array('status' => TRUE, 'data' => $output));
-  exit();
-  
-}
 
 /**
  * Implements node_load
@@ -382,12 +357,35 @@ function tripal_bulk_loader_load($node){
     $r->num_inserted = sizeof(preg_split('/,/',$r->ids_inserted)); 
     $node->inserted_records->{$r->table_inserted_into} = $r;
   }
+
+  // Add exposed field list
+  $node->exposed_fields = array();
+  if ($template) {
+    foreach ($template as $record_id => $record) {
+      foreach ($record['fields'] as $field_id => $field) {
+        if ($field['exposed']) {
+          $node->exposed_fields[] = array(
+            'record_id' => $record_id,
+            'field_id' => $field_id,
+            'title' => $field['title'],
+          );
+        }
+      }
+    }
+    
+    if (empty($node->exposed_fields)) {
+      $node->exposed_fields[] = array();
+    }
+  }
+
   
   // Add constants
-  $sql = 'SELECT * FROM {tripal_bulk_loader_constants} WHERE nid=%d ORDER BY record_id, field_id';
+  $sql = 'SELECT * FROM {tripal_bulk_loader_constants} WHERE nid=%d ORDER BY group_id, record_id, field_id';
   $resource = db_query($sql,$node->nid);
   while ($r = db_fetch_object($resource)) {
-    $node->constants[$r->record_id][$r->field_id] = array(
+    $node->constants[$r->group_id][$r->record_id][$r->field_id] = array(
+      'constant_id' => $r->constant_id,
+      'group_id' => $r->group_id,
       'chado_table'=>$r->chado_table, 
       'chado_field'=>$r->chado_field, 
       'record_id'=>$r->record_id, 
@@ -395,6 +393,9 @@ function tripal_bulk_loader_load($node){
       'value'=>$r->value
     );
   }
+  if (!$node->constants) {
+    $node->constants[] = array();
+  }
   
 	return $node;
 }
@@ -453,228 +454,6 @@ function tripal_bulk_loader_update ($node) {
   
 }
 
-///////////////////////////////////////////////////////////
-// Set Constants Form
-///////////////////////////////////////////////////////////
-
-/**
- * Set constants (exposed fields in template)
- *
- * @param $form_state 
- *   The current state of the form
- * @param $node
- *   The node to set constants for
- *
- * @return
- *   A form array to be rendered by drupal_get_form()
- */
-function tripal_bulk_loader_set_constants_form ($form_state, $node) {
-  $form = array();
-  
-  $form['nid'] = array(
-    '#type' => 'hidden',
-    '#value' => $node->nid
-  );
-  
-  $form['exposed_fields'] = array(
-    '#type' => 'fieldset',
-    '#title' => t('Constant Values'),
-    '#collapsible' => TRUE,
-    '#collapsed' => ($node->template_id) ? FALSE : TRUE,
-    '#prefix' => '<div id="set-constants">',
-    '#suffix' => '</div>',
-  );
-  
-  $form['exposed_fields']['explanation'] = array(
-    '#type' => 'item',
-    '#value' => t('The following fields are constants in the selected template that you need to set values for.')
-  );
-  
-  // Add textifelds for exposed fields of the current template
-  $exposed_fields = FALSE;
-  $indexes = array();
-  if ($node->template_id) {
-    if (isset($node->template)) {
-      foreach ($node->template->template_array as $record_id => $record) {
-        foreach ($record['fields'] as $field_id => $field) {
-          if ($field['exposed']) {
-            $exposed_fields = TRUE;
-            $indexes[$record_id][] = $field_id;
-            
-            switch($field['type']) {
-              case 'table field':
-                $form['exposed_fields'][$record_id.'-'.$field_id] = array(
-                  '#type' => 'textfield',
-                  '#title' => t($field['title']),
-                  '#description' => t($field['exposed_description']),
-                  '#default_value' => (isset($node->constants[$record_id][$field_id]['value'])) ? $node->constants[$record_id][$field_id]['value'] : $field['constant value'],
-                );
-              break;
-              case 'constant':
-                $form['exposed_fields'][$record_id.'-'.$field_id] = array(
-                  '#type' => 'textfield',
-                  '#title' => t($field['title']),
-                  '#description' => t('Enter the case-sensitive value of this constant for your spreadsheet'),
-                  '#default_value' => (isset($node->constants[$record_id][$field_id]['value'])) ? $node->constants[$record_id][$field_id]['value'] : $field['constant value'],
-                );
-              break;
-            }
-            
-            $form['exposed_fields'][$record_id.'-'.$field_id.'-table'] = array(
-              '#type' => 'hidden',
-              '#value' => $record['table'],
-            );
-            $form['exposed_fields'][$record_id.'-'.$field_id.'-field'] = array(
-              '#type' => 'hidden',
-              '#value' => $field['field'],
-            );
-            $form['exposed_fields'][$record_id.'-'.$field_id.'-type'] = array(
-              '#type' => 'hidden',
-              '#value' => $field['type'],
-            );
-          }
-        }
-      }
-    }
-  }
-  $form['template'] = array(
-    '#type' => 'hidden',
-    '#value' => serialize($node->template->template_array)
-  );
-  
-  $form['exposed_fields']['indexes'] = array(
-    '#type' => 'hidden',
-    '#value' => serialize($indexes),
-  );
-  
-  if (!$exposed_fields) {
-    $form['exposed_fields']['explanation'] = array(
-      '#type' => 'item',
-      '#value' => t('There are no exposed fields for this template.')
-    );    
-  }
-  
-  $form['exposed_fields']['submit'] = array(
-    '#type' => 'submit',
-    '#value' => t('Save Changes')
-  );
-  
-  return $form;
-}
-
-/**
- * Validate that the values entered exist in the database
- * if indicated in hte template array
- */
-function tripal_bulk_loader_set_constants_form_validate ($form_state, $form) {
-
-  $template = unserialize($form_state['template']['#value']);
-  $indexes = unserialize($form_state['exposed_fields']['indexes']['#value']);
-  
-  foreach ($indexes as $record_id => $array) {
-    foreach($array as $field_id) {
-      if ($template[$record_id]['fields'][$field_id]['exposed_validate']) {
-        $result = db_fetch_object(db_query(
-          "SELECT 1 as valid FROM %s WHERE %s='%s'",
-          $template[$record_id]['table'],
-          $template[$record_id]['fields'][$field_id]['field'],
-          $form_state['exposed_fields'][$record_id.'-'.$field_id]['#value']
-        ));
-
-        if (!$result->valid) {
-          $msg = 'A '.$form_state['exposed_fields'][$record_id.'-'.$field_id]['#title'].' of "'.$form_state['exposed_fields'][$record_id.'-'.$field_id]['#value'].'" must already exist!';
-          form_set_error($record_id.'-'.$field_id, $msg);
-        } else {
-          drupal_set_message('Confirmed a '.$form_state['exposed_fields'][$record_id.'-'.$field_id]['#title'].' of "'.$form_state['exposed_fields'][$record_id.'-'.$field_id]['#value'].'" already exists.');
-        }
-      }
-    }
-  }
-  
-}
-
-/**
- * Insert/update the constants associated with this node
- */
-function tripal_bulk_loader_set_constants_form_submit ($form, $form_state) {
-
-	// Insert/Update constants
-  $template = unserialize($form_state['values']['template']);
-  $indexes = unserialize($form_state['values']['indexes']);
-  
-  foreach ($indexes as $record_id => $array) {
-    foreach($array as $field_id) {	
-      tripal_bulk_loader_update_constant(
-        $form_state['values'][nid], 
-        $form_state['values'][$record_id.'-'.$field_id.'-table'], 
-        $form_state['values'][$record_id.'-'.$field_id.'-field'],
-        $record_id,
-        $field_id,
-        $form_state['values'][$record_id.'-'.$field_id]
-        );
-    }
-  }
-  
-}
-
-/**
- * Inserts/Updates a tripal bulk loading job constant
- *
- * @param $nid
- *   The node ID of the the tripal bulk loading job the constant is associated with
- * @param $table
- *   The chado table the constant is associated with
- * @param $field
- *   The chado field the constant is associated with
- * @param $record_id
- *   The index in the template array for this record
- * @param $field_id
- *   The index in the template array for this field
- *
- * NOTE: $template_array[$record_id]['table'] = $table and $template_array[$record_id]['fields'][$field_id]['field'] = $field
- *   both are included as a means of double-checking the constant still is still in thesame place in the template array.
- *   For example, that the template was not edited and the records moved around after the job was submitted but before it was run.
- *
- * @return
- *   On success it returns the object (with primary key if inserted);
- *   on failure it returns FALSE
- */
-function tripal_bulk_loader_update_constant ($nid, $table, $field, $record_id, $field_id, $value) {
-  
-  $record = array(
-    'nid'=>$nid, 
-    'chado_table'=>$table, 
-    'chado_field'=>$field, 
-    'record_id'=>$record_id, 
-    'field_id'=>$field_id, 
-    'value'=>$value
-  );
-  
-  // Check to see if already exists
-  $exists = db_fetch_object(db_query(
-    "SELECT constant_id FROM {tripal_bulk_loader_constants} WHERE nid=%d AND record_id=%d AND field_id=%d",
-    $record['nid'],
-    $record['record_id'],
-    $record['field_id']
-  ));
-  if ($exists->constant_id) {
-    $record['constant_id'] = $exists->constant_id;
-    $status = drupal_write_record('tripal_bulk_loader_constants',$record,'constant_id');
-    if ($status) {
-      return $record;
-    } else {
-      return FALSE;
-    }
-  } else {
-    $status = drupal_write_record('tripal_bulk_loader_constants',$record);
-    if ($status) {
-      return $record;
-    } else {
-      return FALSE;
-    }
-  }
-}
-
 ///////////////////////////////////////////////////////////
 
 /**

base/tripal_core/mviews.php

@@ -413,30 +413,24 @@ function tripal_mviews_form(&$form_state = NULL,$mview_id = NULL){
 function tripal_mviews_form_submit($form, &$form_state){
    
    $action = $form_state['values']['action'];
+   $mview_id = $form_state['values']['mview_id'];
+   $name = $form_state['values']['name'];
+   $mv_table = $form_state['values']['mv_table'];
+   $mv_specs = $form_state['values']['mv_specs'];
+   $indexed = $form_state['values']['indexed'];
+   $query = $form_state['values']['mvquery'];
+   $special_index = $form_state['values']['special_index'];
 
    if(strcmp($action,'Update')==0){
-      $record = new stdClass();
-      $record->mview_id = $form_state['values']['mview_id'];
-      $record->name = $form_state['values']['name'];
-      $record->mv_table = $form_state['values']['mv_table'];
-      $record->mv_specs = $form_state['values']['mv_specs'];
-      $record->indexed = $form_state['values']['indexed'];
-      $record->query = $form_state['values']['mvquery'];
-      $record->special_index = $form_state['values']['special_index'];
-
-      // add the record to the tripal_mviews table and if successful
-      // create the new materialized view in the chado schema
-      if(drupal_write_record('tripal_mviews',$record,'mview_id')){
-         drupal_set_message('View updated successfullly');
-      } else {
-         drupal_set_message('View update failed');
-      }
+      // updating the materialized view consits of deleting the old entry
+      // and readding.  This is necessary because a change to any of the fields
+      // other than the query changes the nature of table so it needs to be 
+      // rebuilt
+      tripal_mviews_action ('delete',$mview_id);
+      tripal_add_mview ($name, 'tripal_core',$mv_table, $mv_specs,$indexed,$query,$special_index);
    }
    else if(strcmp($action,'Add')==0){
-      tripal_add_mview ($form_state['values']['name'], 'tripal_core',
-         $form_state['values']['mv_table'], $form_state['values']['mv_specs'],
-         $form_state['values']['indexed'], $form_state['values']['mvquery'],
-         $form_state['values']['special_index']);
+      tripal_add_mview ($name, 'tripal_core',$mv_table, $mv_specs,$indexed,$query,$special_index);
    }
    else {
         drupal_set_message("No action performed.");

base/tripal_cv/views/cvterm.views.inc

@@ -88,7 +88,7 @@ function retrieve_cvterm_views_data() {
        'handler' => 'views_handler_sort',
      ),
      'filter' => array(
-       'handler' => 'views_handler_filter_chado_select_cvterm_name',
+       'handler' => 'views_handler_filter_string',
      ),
      'argument' => array(
        'handler' => 'views_handler_argument_string',
@@ -123,7 +123,7 @@ function retrieve_cvterm_views_data() {
        'click sortable' => TRUE,
      ),
      'filter' => array(
-       'handler' => 'views_handler_filter_chado_boolean',
+       'handler' => 'views_handler_filter_string',
        'label' => t('Is Obsolete?'),
        'type' => 'yes-no',
      ),

base/tripal_db/views/handlers/views_handler_field_dbxref_accession_link.inc

@@ -1 +0,0 @@
-../../../tripal_core/views/handlers/views_handler_field_dbxref_accession_link.inc

base/tripal_db/views/handlers/views_handler_field_dbxref_accession_link.inc

@@ -0,0 +1 @@
+../../../tripal_core/views/handlers/views_handler_field_dbxref_accession_link.inc

base/tripal_feature/views/feature.views.inc

@@ -142,7 +142,7 @@
     ),
   );
   // if joined to the node table add a "Link to Node" option for the field  
-  if (!$database) {
+  if (!$database && module_exists('tripal_views')) {
     $data['feature']['name']['field']['handler'] = 'views_handler_field_node_optional';
   }
 
@@ -165,7 +165,7 @@
     ),
   );
   // if joined to the node table add a "Link to Node" option for the field  
-  if (!$database) {
+  if (!$database && module_exists('tripal_views')) {
     $data['feature']['uniquename']['field']['handler'] = 'views_handler_field_node_optional';
   }
 
@@ -288,6 +288,7 @@
       'help' => t('Display a given type of relationships including the current feature.'),
       'handler' => 'views_handler_field_chado_rel_by_type',
     ),
-  );  
+  ); 
+  
   return $data;
  }

base/tripal_feature/views/handlers/views_handler_field_readable_date.inc

@@ -1 +0,0 @@
-../../../tripal_core/views/handlers/views_handler_field_readable_date.inc

base/tripal_feature/views/handlers/views_handler_field_readable_date.inc

@@ -0,0 +1 @@
+../../../tripal_core/views/handlers/views_handler_field_readable_date.inc

base/tripal_library/views/handlers/views_handler_field_readable_date.inc

@@ -1 +0,0 @@
-../../../tripal_core/views/handlers/views_handler_field_readable_date.inc

base/tripal_library/views/handlers/views_handler_field_readable_date.inc

@@ -0,0 +1 @@
+../../../tripal_core/views/handlers/views_handler_field_readable_date.inc

base/tripal_organism/views/organism.views.inc

@@ -86,7 +86,7 @@ function retrieve_organism_views_data() {
   // Calculated Field: Node ID
   //  use custom field handler to query drupal for the node ID
   //  this is only needed if chado is in a separate database from drupal
-  if($database){
+  if($database && module_exists('tripal_views')){
 	  $data['organism']['organism_nid'] = array(
   	  'title' => 'Node ID',
     	'help' => 'This is the node ID of this organism. It can be used as a link to the node.',
@@ -125,14 +125,14 @@ function retrieve_organism_views_data() {
        'handler' => 'views_handler_sort',
      ),
      'filter' => array(
-       'handler' => 'views_handler_filter_chado_select_string',
+       'handler' => 'views_handler_filter_string',
      ),
      'argument' => array(
        'handler' => 'views_handler_argument_string',
      ),
   );
   // if joined to the node table add a "Link to Node" option for the field  
-  if (!$database) {
+  if (!$database && module_exists('tripal_views')) {
     $data['organism']['abbreviation']['field']['handler'] = 'views_handler_field_node_optional';
   }
 
@@ -148,7 +148,7 @@ function retrieve_organism_views_data() {
        'handler' => 'views_handler_sort',
      ),
      'filter' => array(
-       'handler' => 'views_handler_filter_chado_select_string',
+       'handler' => 'views_handler_filter_string',
      ),
      'argument' => array(
        'handler' => 'views_handler_argument_string',
@@ -167,7 +167,7 @@ function retrieve_organism_views_data() {
        'handler' => 'views_handler_sort',
      ),
      'filter' => array(
-       'handler' => 'views_handler_filter_chado_select_string',
+       'handler' => 'views_handler_filter_string',
      ),
      'argument' => array(
        'handler' => 'views_handler_argument_string',
@@ -186,14 +186,14 @@ function retrieve_organism_views_data() {
        'handler' => 'views_handler_sort',
      ),
      'filter' => array(
-       'handler' => 'views_handler_filter_chado_select_string',
+       'handler' => 'views_handler_filter_string',
      ),
      'argument' => array(
        'handler' => 'views_handler_argument_string',
      ),
   );
   // if joined to the node table add a "Link to Node" option for the field  
-  if (!$database) {
+  if (!$database && module_exists('tripal_views')) {
     $data['organism']['common_name']['field']['handler'] = 'views_handler_field_node_optional';
   }
 

base/tripal_views/tripal_views.module

@@ -4,7 +4,9 @@ require_once "tripal_views_integration.inc";
 require_once "tripal_views.views.inc";
 
 /**
+ * Implements hook_menu()
  *
+ * Purpose: this hook provides details about new menu items added by this module
  *
  * @ingroup tripal_views
  */
@@ -20,8 +22,8 @@ function tripal_views_menu() {
    );
 
  	$items['admin/tripal/views/integration'] = array(
-     'title' => t('Views Integration'),
-     'description' => t('Allows you to select existing materialized views and provide details for integration with Drupal Views.'),
+     'title' => t('Integrated Tables'),
+     'description' => t('Provide a list of all integrated tables and allows for adding new tables or editing already integrated tables.'),
      'page callback' => 'tripal_views_integration_setup_list',
      'access arguments' => array('manage tripal_views_integration'),
      'type' => MENU_NORMAL_ITEM,
@@ -55,9 +57,9 @@ function tripal_views_menu() {
    return $items;
 }
 /**
- *  Set the permission types that the chado module uses.  Essentially we
- *  want permissionis that protect creation, editing and deleting of chado
- *  data objects
+ * Implements hook_views_api()
+ * 
+ * Purpose: Set the permission types that the chado module uses.  
  *
  * @ingroup tripal_views
  */
@@ -81,6 +83,10 @@ function tripal_views_views_api() {
    );
 }
 /**
+ * Implements hook_theme()
+ *
+ * Purpose: this hook provides details about themable objects added by 
+ *   this module
  *
  * @ingroup tripal_views
  */

base/tripal_views/tripal_views.views.inc

@@ -376,6 +376,14 @@ function tripal_views_add_node_ids_to_view (&$view) {
 	return $view;
 }
 /**
+ * Generates a dynamic data array for Views
+ *
+ * Purpose: This function is a hook used by the Views module. It populates and
+ *   returns a data array that specifies for the Views module the base table,
+ *   the tables it joins with and handlers.  The data array is populated
+ *   using the data stored in the tripal_views tables.
+ *
+ * @return a data array formatted for the Views module
  *
  * @ingroup tripal_views
  */
@@ -621,6 +629,18 @@ function tripal_views_views_plugins() {
  * Implementation of hook_views_pre_view().
  */
 function tripal_views_views_pre_view(&$view,&$display_id,&$args){
+
+   // merge the $_GET and $_POST into the $_GET. This is because
+   // Views and Views Data Export modules only uses the $_GET variable but
+   // file uploads require $_POST. We need to make sure these two modules
+   // have access to everything needed for this view to work properlys
+   $_GET = array_merge($_GET, $_POST);
+
+
+   // we want to add to the bottom of the views the form for downloading 
+   // results in other formats (e.g. Excel, FASTA, CSV, etc.).  The Views Data
+   // Export module provides small images at the bottom, but we want to provide
+   // a more intutitive interface for getting different file formats
    $form = drupal_get_form('tripal_views_data_export_download_form',$view,$display_id,$args);
    $view->attachment_after = $form;
 }

base/tripal_views/tripal_views_integration.inc

@@ -54,6 +54,12 @@ function tripal_views_description_page() {
   return $text;
 }
 /**
+ * Purpose: Generates a themable table containing the list of integrated tables
+ *   The look-and-feel of the table can be altered by overriding the theme for
+ *   tables.
+ *
+ * @return
+ *    a themed HTML table   
  *
  * @ingroup tripal_views_integration
  */
@@ -86,6 +92,12 @@ function tripal_views_integration_setup_list(){
 }
 
 /**
+ * Purpose: Deletes integration of a table with the Views module.  This 
+ *   function is meant to be called from a menu item. After completion it
+ *   redirects the user to the views intergation page.
+ *
+ * @param $setup_id
+ *    the unique setup id for the integrated table   
  *
  * @ingroup tripal_views_integration
  */
@@ -96,6 +108,19 @@ function tripal_views_integration_delete($setup_id){
 }
 
 /**
+ * Purpose: defines the web form used for specifing the base table, joins and
+ *   handlers when integrating a table with views.  This form is used for both
+ *   creating a new record and editing an existing record.
+ *
+ * @param &$form_state
+ *    The form state which is passed automatically by drupal  
+ *
+ * @param $setup_id
+ *    The unique setup for an integrated table.  This value is only set when
+ *    the form is used for updating an existing record.  
+ *
+ * @return
+ *    A proper Drupal form associative array.
  *
  * @ingroup tripal_views_integration
  */
@@ -688,6 +713,13 @@ function tripal_views_integration_form(&$form_state, $setup_id = NULL){
 
 
 /**
+ * Purpose: validates the tripal_views_integration_form after submission
+ *
+ * @param $form
+ *    The form object which is passed automatically by drupal  
+ *
+ * @param &$form_state
+ *    The form state pbject which is passed automatically by drupal  
  *
  * @ingroup tripal_views_integration
  */
@@ -709,6 +741,14 @@ function tripal_views_integration_form_validate($form, &$form_state){
 }
 
 /**
+ * Purpose: inserts or updates the record in the tripal views integration
+ *   tables.  This function is only called if validation is passed. 
+ *
+ * @param $form
+ *    The form object which is passed automatically by drupal  
+ *
+ * @param &$form_state
+ *    The form state pbject which is passed automatically by drupal  
  *
  * @ingroup tripal_views_integration
  */
@@ -717,6 +757,8 @@ function tripal_views_integration_form_submit($form, &$form_state){
    $mview_id = $form_state['values']['mview_id'];
    $table_name = $form_state['values']['table_name'];
    $setup_id = $form_state['values']['setup_id'];
+   $priority = $form_state['values']['row_priority'];
+   $comment = $form_state['values']['row_description'];
 
    // get details about this mview
    if ($mview_id) {
@@ -742,19 +784,18 @@ function tripal_views_integration_form_submit($form, &$form_state){
        'mview_id' => $mview_id,
        'table_name' => $mview->mv_table,
        'name' => $name,
-       'priority' => $form_state['values']['row_priority'],
-       'comment' => $form_state['values']['row_description'],
-       
+       'priority' => $priority,
+       'comment' => $comment,       
       );
    } 
    // if a chado table then...
-   if($table_name){
-            // build the record for insert/update
+   else {
+      // build the record for insert/update
       $tripal_views_record = array(
        'table_name' => $table_name,
        'name' => $name,
-       'priority' => $form_state['values']['row_priority'],
-       'comment' => $form_state['values']['row_description'],
+       'priority' => $priority,
+       'comment' => $comment,
       );
    }
 
@@ -772,8 +813,6 @@ function tripal_views_integration_form_submit($form, &$form_state){
       }
    }
    
-
-   
    // if this is an update then clean out the existing joins and handlers so we can add new ones
    if($setup_id){  
       db_query("DELETE FROM {tripal_views_field} WHERE setup_id = %d",$setup_id);
@@ -842,6 +881,12 @@ function tripal_views_integration_form_submit($form, &$form_state){
 }
 
 /**
+ * Purpose: this function queries all modules currently enabled on the site
+ *   looking for custom handlers and returns a list of all available handerls.
+ *   The base View handlers are also included.
+ *
+ * @return
+ *   Returns an array of handler names
  *
  * @ingroup tripal_views_integration
  */

base/tripal_views/views/handlers/views_handler_filter_file_upload.inc

@@ -1,11 +1,10 @@
 <?php
 
-require_once "/var/www/sites/all/modules/views/handlers/views_handler_filter_string.inc";
 /**
- * Purpose: This Handler provides a file upload field.
+ * Purpose: This Handler provides a file upload field by extending the
+ * views_handler_filter object.
  *
- * @ingroup views_filter_handlers
- * @ingroup tripal_core
+ * @ingroup tripal_views_integration
  */
 class views_handler_filter_file_upload extends views_handler_filter {
 
@@ -28,7 +27,7 @@ class views_handler_filter_file_upload extends views_handler_filter {
     
     // we'll provide a single text area for this field.
     // in the exposed_form function we'll add in the file upload button
-    $form['value'] = array(
+    $form['value'][$this->options['field'] . '_textarea']  = array(
        '#type' => 'textarea',
        '#title' => $this->options['expose']['label'],
        '#default_value' => $this->value,
@@ -36,6 +35,13 @@ class views_handler_filter_file_upload extends views_handler_filter {
        '#description' => t('Provide search values for ' . $this->options['expose']['label'] . 
          '. Please place each search item on a separate line or separated by commas.'),
     ); 
+    $form['value'][$this->options['field'] . '_upload'] = array(
+      '#type' => 'file',
+      '#title' => '',
+      '#description' => t('Upload a file to provide search values for ' . $this->options['expose']['label'] . 
+         '. Please place each search item on a separate line. If a file is uploaded but values are present '.
+         'in the text box above, the values in the text box will be ignored.'),
+    );
   }
 
   /** 
@@ -46,10 +52,12 @@ class views_handler_filter_file_upload extends views_handler_filter {
 
   }
   /**
-  * Ensures the select list gets rendered when the filter is exposed
+  * Ensures the upload field gets rendered when the filter is exposed. It also
+  * changes the form type from a GET to a POST so that file uploads will work.
   */
   function exposed_form(&$form, &$form_state) {
 
+    // don't do anything if the form isn't exposed.
     if (empty($this->options['exposed'])) {
       return;
     }
@@ -58,13 +66,10 @@ class views_handler_filter_file_upload extends views_handler_filter {
     $value = $this->options['expose']['identifier'];
     $this->value_form($form, $form_state);
     $form[$value] = $form['value'];
-    unset($form[$value]['#title']);
-    $form[$this->options['field'] . '_upload'] = array(
-      '#type' => 'file',
-      '#title' => '',
-      '#description' => t('Upload a file to provide search values for ' . $this->options['expose']['label'] . 
-         '. Please place each search item on a separate line.'),
-    );
+    unset($form[$value][$this->options['field'] . '_textarea']['#title']);
+
+    // since this is an exposed form we want to enable file uploads by
+    // setting the 'enctype' attribute and the method to POST
     $form['#attributes']['enctype'] = 'multipart/form-data';
     $form['#method'] = 'POST';
     $this->exposed_translate($form[$value], 'value');
@@ -78,7 +83,7 @@ class views_handler_filter_file_upload extends views_handler_filter {
      $holders = array();
      $values = array();
 
-     // get the file upload info if one has been provided
+     // get the file upload content if one has been provided
      $file = file_save_upload($this->field.'_upload',array());
      if($file){
         $fh = fopen($file->filepath,'r');

extensions/tripal_analysis_blast/theme/tripal_feature/tripal_feature_blast_results.tpl.php

@@ -71,7 +71,7 @@ if(count($blast_results_list) > 0){
 		foreach($hits_array AS $hit) { 
          $class = 'tripal-table-odd-row tripal_analysis_blast-table-odd-row';
          if($i % 2 == 0 ){
-            $class = 'tripal-table-even-row tripal_analysis_blast-table-odd-row';
+            $class = 'tripal-table-even-row tripal_analysis_blast-table-even-row';
          }?>
          <tr class="<?php print $class ?> tripal_analysis_blast-result-first-row">
             <td><?php print $i+1 ?>.</td>

extensions/tripal_analysis_blast/tripal_analysis_blast.module

@@ -9,7 +9,7 @@ require_once "includes/parse_blast_XML.inc";
  ******************************************************************************/
 function tripal_analysis_blast_init(){
 	// Add javascript and style sheet
-	drupal_add_css(drupal_get_path('module', 'tripal_analysis_blast') .'/theme/css/tripal_analysis_blast.css');
+	drupal_add_css(drupal_get_path('module', 'tripal_analysis_blast') .'/theme/css/tripal_analysis_blast.css', 'theme');
 	drupal_add_js(drupal_get_path('module', 'tripal_analysis_blast') .'/theme/js/tripal_analysis_blast.js');
 }
 

extensions/tripal_analysis_go/tripal_analysis_go.module

@@ -4,7 +4,7 @@ require_once "includes/gaf_loader.inc";
 
 function tripal_analysis_go_init(){
    // Add style sheet
-   drupal_add_css(drupal_get_path('module', 'tripal_analysis_go').'/theme/css/tripal_analysis_go.css');
+   drupal_add_css(drupal_get_path('module', 'tripal_analysis_go').'/theme/css/tripal_analysis_go.css', 'theme');
 
    drupal_add_js(drupal_get_path('module', 'tripal_analysis_go').'/theme/js/tripal_analysis_go.js');
 }
@@ -390,7 +390,7 @@ function tripal_analysis_go_cv_chart($chart_id){
         )
      ",
      type             => 'p',
-     size             => '550x175',
+     size             => '650x260',
      title            => $title,
   );
   return $options;

extensions/tripal_analysis_interpro/node-chado_analysis_interpro.tpl.php

@@ -83,7 +83,7 @@ if (Drupal.jsEnabled) {
 <div id="tripal_analysis_interpro_details" class="tripal_details">
 
    <!-- Basic Details Theme -->
-   <?php include('tripal_analysis_interpro/tripal_analysis_interpro_base.tpl.php'); ?>
+   <?php include('theme/tripal_analysis_interpro/tripal_analysis_interpro_base.tpl.php'); ?>
 
    <?php print $content ?>
 </div>

extensions/tripal_analysis_interpro/tripal_analysis_interpro.module

@@ -9,7 +9,7 @@ require_once "includes/parseInterpro.inc";
 function tripal_analysis_interpro_init(){
 
 	// Add javascript and style sheet
-   drupal_add_css(drupal_get_path('module', 'tripal_analysis_interpro').'/theme/css/tripal_analysis_interpro.css');
+   drupal_add_css(drupal_get_path('module', 'tripal_analysis_interpro').'/theme/css/tripal_analysis_interpro.css', 'theme');
    drupal_add_js( drupal_get_path('module', 'tripal_analysis_interpro').'/theme/js/tripal_analysis_interpro.js');
 }
 /*******************************************************************************
@@ -428,7 +428,7 @@ function tripal_analysis_interpro_preprocess_tripal_feature_interpro_results(&$v
 
    // we don't know how many analysis are the old style HTML version and how many are the newer
    // XML format.  So, to be backwards compatible we should get both.
-   $feature->tripal_analysis_interpro->results->html = tripal_get_interpro_HTML_results($feature->feature_id);
+  // $feature->tripal_analysis_interpro->results->html = tripal_get_interpro_HTML_results($feature->feature_id);
 	$feature->tripal_analysis_interpro->results->xml = tripal_get_interpro_XML_results($feature->feature_id);
 }
 

extensions/tripal_analysis_kegg/tripal_analysis_kegg.module

@@ -8,9 +8,9 @@
 function tripal_analysis_kegg_init(){
    // add the tripal_analysis_kegg JS and CSS
    drupal_add_js(drupal_get_path('module', 'tripal_analysis_kegg').'/theme/js/tripal_analysis_kegg.js');
-   drupal_add_css(drupal_get_path('module', 'tripal_analysis_kegg').'/theme/css/tripal_analysis_kegg.css');
+   drupal_add_css(drupal_get_path('module', 'tripal_analysis_kegg').'/theme/css/tripal_analysis_kegg.css', 'theme');
    // add the jsTree JS and CSS
-   drupal_add_css(drupal_get_path('theme', 'tripal').'/js/jsTree/source/tree_component.css');
+   drupal_add_css(drupal_get_path('theme', 'tripal').'/js/jsTree/source/tree_component.css', 'theme');
    drupal_add_js (drupal_get_path('theme', 'tripal').'/js/jsTree/source/_lib.js');
    drupal_add_js (drupal_get_path('theme', 'tripal').'/js/jsTree/source/tree_component.js');
 }

extensions/tripal_analysis_unigene/node-chado_analysis_unigene.tpl.php

@@ -86,7 +86,7 @@ if (Drupal.jsEnabled) {
 <div id="tripal_analysis_unigene_details" class="tripal_details">
 
    <!-- Basic Details Theme -->
-   <?php include('tripal_analysis_unigene/tripal_analysis_unigene_base.tpl.php'); ?>
+   <?php include('theme/tripal_analysis_unigene/tripal_analysis_unigene_base.tpl.php'); ?>
 
    <?php print $content ?>
 </div>

extensions/tripal_analysis_unigene/tripal_analysis_unigene.module

@@ -2,7 +2,7 @@
 
 function tripal_analysis_unigene_init(){
    // Add style sheet
-   drupal_add_css(drupal_get_path('module', 'tripal_analysis_unigene').'/theme/css/tripal_analysis_unigene.css');
+   drupal_add_css(drupal_get_path('module', 'tripal_analysis_unigene').'/theme/css/tripal_analysis_unigene.css', 'theme');
 }
 /**
 *