Merge branch '6.x-1530236-multiple_sets_constants-bulk_loader' into 6.x-0.4-dev

base/tripal_bulk_loader/tripal_bulk_loader.constants.inc

@@ -0,0 +1,547 @@
+<?php
+
+/**
+ * Inserts/Updates a tripal bulk loading job constant
+ *
+ * @param $nid
+ *   The node ID of the the tripal bulk loading job the constant is associated with
+ * @param $table
+ *   The chado table the constant is associated with
+ * @param $field
+ *   The chado field the constant is associated with
+ * @param $record_id
+ *   The index in the template array for this record
+ * @param $field_id
+ *   The index in the template array for this field
+ *
+ * NOTE: $template_array[$record_id]['table'] = $table and $template_array[$record_id]['fields'][$field_id]['field'] = $field
+ *   both are included as a means of double-checking the constant still is still in thesame place in the template array.
+ *   For example, that the template was not edited and the records moved around after the job was submitted but before it was run.
+ *
+ * @return
+ *   On success it returns the object (with primary key if inserted);
+ *   on failure it returns FALSE
+ */
+function tripal_bulk_loader_update_constant ($nid, $group_id, $table, $field, $record_id, $field_id, $value) {
+  
+  $record = array(
+    'nid'=>$nid, 
+    'group_id' => $group_id,
+    'chado_table'=>$table, 
+    'chado_field'=>$field, 
+    'record_id'=>$record_id, 
+    'field_id'=>$field_id, 
+    'value'=>$value
+  );
+  
+  // Check to see if already exists
+  $exists = db_fetch_object(db_query(
+    "SELECT constant_id FROM {tripal_bulk_loader_constants} WHERE nid=%d AND record_id=%d AND field_id=%d AND group_id=%d",
+    $record['nid'],
+    $record['record_id'],
+    $record['field_id'],
+    $record['group_id']
+  ));
+  if ($exists->constant_id) {
+    $record['constant_id'] = $exists->constant_id;
+    $status = drupal_write_record('tripal_bulk_loader_constants',$record,'constant_id');
+    if ($status) {
+      return $record;
+    } else {
+      return FALSE;
+    }
+  } else {
+    $status = drupal_write_record('tripal_bulk_loader_constants',$record);
+    if ($status) {
+      return $record;
+    } else {
+      return FALSE;
+    }
+  }
+}
+
+function tripal_bulk_loader_has_exposed_fields ($node) {
+
+  // exposed fields isn't set
+  if (!isset($node->exposed_fields)) {
+    return FALSE;
+  }
+  
+  // exposed fields has at least one element
+  if (sizeof($node->exposed_fields) == 1) {
+    // need to check if single element is an empty array
+    $element = reset($node->exposed_fields);
+    if ($element) {
+      return TRUE;
+    } else {
+      return FALSE;
+    }
+  } elseif (sizeof($node->exposed_fields) > 1) {
+    return TRUE;
+  } else {
+    return FALSE;
+  }
+  
+  return FALSE;
+}
+
+///////////////////////////////////////////////////////////
+// Set Constants Form (on Bulk Loader Node)
+///////////////////////////////////////////////////////////
+
+/**
+ * Set constants (exposed fields in template)
+ *
+ * @param $form_state 
+ *   The current state of the form
+ * @param $node
+ *   The node to set constants for
+ *
+ * @return
+ *   A form array to be rendered by drupal_get_form()
+ */
+function tripal_bulk_loader_set_constants_form ($form_state, $node) {
+  $form = array();
+  
+  $form['nid'] = array(
+    '#type' => 'hidden',
+    '#value' => $node->nid
+  );
+  
+  if (!tripal_bulk_loader_has_exposed_fields($node)) {
+    return $form;
+  }
+  
+  $form['exposed_array'] = array(
+    '#type' => 'hidden',
+    '#value' => serialize($node->exposed_fields),
+  );
+  
+  $form['exposed_fields'] = array(
+    '#type' => 'fieldset',
+    '#title' => t('Constant Values'),
+    '#collapsible' => TRUE,
+    '#collapsed' => ($node->template_id) ? FALSE : TRUE,
+    '#prefix' => '<div id="set-constants">',
+    '#suffix' => '</div>',
+  );
+
+  // Display table of already added constant sets with the ability to re-arrange and delete
+  if (sizeof($node->constants) > 0) {
+    $form['exposed_fields']['explanation-1'] = array(
+      '#type' => 'item',
+      '#value' => t('You have already added constants to this bulk loading job. Each '
+        .'row in the following table represents a set of constants. Each set will be used '
+        .'to load your spreadsheet with the specified template resulting in the each record '
+        .'in the template to be loaded x number of times where there are x sets of '
+        .'constants (rows in the following table).')
+    );
+    
+    $form['exposed_fields']['existing'] = array(
+      '#tree' => TRUE,
+    );
+    
+    foreach ($node->constants as $set) {
+
+      foreach ($set as $record) {
+        foreach ($record as $field) {
+          $index = $field['record_id'].'-'.$field['field_id'];
+          $group = $field['group_id'];
+          $form['exposed_fields']['existing'][$group][$index] = array(
+            '#type' => 'markup',
+            '#value' => $field['value'],
+          );
+        }
+      }
+
+      $form['exposed_fields']['existing'][$group]['delete'] = array(
+        '#type' => 'markup',
+        '#value' => l('Edit', 'node/'. $node->nid . '/constants/'.$group.'/edit') .'<br />'.
+          l('Delete', 'node/'. $node->nid . '/constants/'.$group.'/delete'),
+      );
+                  
+    }
+  }
+  
+  $form['exposed_fields']['new'] = array(
+    '#type' => 'fieldset',
+    '#title' => t('New set of Constants'),
+  );
+    
+  $form['exposed_fields']['new']['explanation-2'] = array(
+    '#type' => 'item',
+    '#value' => t('The following fields are constants in the selected template that you need to set values for.')
+  );
+  
+  // Add textifelds for exposed fields of the current template
+  $exposed_fields = FALSE;
+  $indexes = array();
+  if (tripal_bulk_loader_has_exposed_fields($node)) {
+    foreach ($node->exposed_fields as $exposed_index) {
+      
+      $record_id = $exposed_index['record_id'];
+      $field_id = $exposed_index['field_id'];
+      $field = $node->template->template_array[$record_id]['fields'][$field_id];
+      
+      if ($field['exposed']) {
+        $exposed_fields = TRUE;
+        $indexes[$record_id][] = $field_id;
+        
+        switch($field['type']) {
+          case 'table field':
+            $form['exposed_fields']['new'][$record_id.'-'.$field_id] = array(
+              '#type' => 'textfield',
+              '#title' => t($field['title']),
+              '#description' => t($field['exposed_description']),
+              '#default_value' => (isset($node->constants[$record_id][$field_id]['value'])) ? $node->constants[$record_id][$field_id]['value'] : $field['constant value'],
+            );
+          break;
+          case 'constant':
+            $form['exposed_fields']['new'][$record_id.'-'.$field_id] = array(
+              '#type' => 'textfield',
+              '#title' => t($field['title']),
+              '#description' => t('Enter the case-sensitive value of this constant for your spreadsheet'),
+              '#default_value' => (isset($node->constants[$record_id][$field_id]['value'])) ? $node->constants[$record_id][$field_id]['value'] : $field['constant value'],
+            );
+          break;
+        }
+        
+        $form['exposed_fields']['new'][$record_id.'-'.$field_id.'-table'] = array(
+          '#type' => 'hidden',
+          '#value' => $record['table'],
+        );
+        $form['exposed_fields']['new'][$record_id.'-'.$field_id.'-field'] = array(
+          '#type' => 'hidden',
+          '#value' => $field['field'],
+        );
+        $form['exposed_fields']['new'][$record_id.'-'.$field_id.'-type'] = array(
+          '#type' => 'hidden',
+          '#value' => $field['type'],
+        );
+
+      }
+    }
+  }
+  $form['template'] = array(
+    '#type' => 'hidden',
+    '#value' => serialize($node->template->template_array)
+  );
+  
+  $form['exposed_fields']['new']['indexes'] = array(
+    '#type' => 'hidden',
+    '#value' => serialize($indexes),
+  );
+  
+  if (!$exposed_fields) {
+    $form['exposed_fields']['new']['explanation'] = array(
+      '#type' => 'item',
+      '#value' => t('There are no exposed fields for this template.')
+    );    
+  }
+  
+  $form['exposed_fields']['new']['submit-2'] = array(
+    '#type' => 'submit',
+    '#name' => 'add_constant',
+    '#value' => t('Add Constant Set')
+  );
+  
+  return $form;
+}
+
+/**
+ * Validate that the values entered exist in the database
+ * if indicated in hte template array
+ */
+function tripal_bulk_loader_set_constants_form_validate ($form, $form_state) {
+
+  $template = unserialize($form_state['values']['template']);
+  $indexes = unserialize($form_state['values']['indexes']);
+
+  $op = $form_state['values'][ $form_state['clicked_button']['#name'] ];
+  if (strcmp('Add Constant Set',$op) == 0) {
+      foreach ($indexes as $record_id => $array) {
+        foreach($array as $field_id) {
+          if ($template[$record_id]['fields'][$field_id]['exposed_validate']) {
+            $result = db_fetch_object(db_query(
+              "SELECT 1 as valid FROM %s WHERE %s='%s'",
+              $template[$record_id]['table'],
+              $template[$record_id]['fields'][$field_id]['field'],
+              $form_state['values'][$record_id.'-'.$field_id]
+            ));
+    
+            if (!$result->valid) {
+              $msg = 'A '.$form['exposed_fields']['new'][$record_id.'-'.$field_id]['#title'].' of "'.$form['exposed_fields']['new'][$record_id.'-'.$field_id]['#value'].'" must already exist!';
+              form_set_error($record_id.'-'.$field_id, $msg);
+            } else {
+              drupal_set_message('Confirmed a '.$form['exposed_fields']['new'][$record_id.'-'.$field_id]['#title'].' of "'.$form['exposed_fields']['new'][$record_id.'-'.$field_id]['#value'].'" already exists.');
+            }
+          }
+        }
+      }
+  }
+  
+}
+
+/**
+ * Insert/update the constants associated with this node
+ */
+function tripal_bulk_loader_set_constants_form_submit ($form, $form_state) {
+
+	// Insert/Update constants
+  $template = unserialize($form_state['values']['template']);
+  $indexes = unserialize($form_state['values']['indexes']);
+  
+  $op = $form_state['values'][ $form_state['clicked_button']['#name'] ];
+  if (strcmp('Add Constant Set',$op) == 0) {
+      $max_group = db_fetch_object(db_query("SELECT max(group_id) as value FROM tripal_bulk_loader_constants WHERE nid=%d",$form_state['values']['nid']));
+      foreach ($indexes as $record_id => $array) {
+        foreach($array as $field_id) {	
+          tripal_bulk_loader_update_constant(
+            $form_state['values']['nid'], 
+            $max_group->value+1,
+            $form_state['values'][$record_id.'-'.$field_id.'-table'], 
+            $form_state['values'][$record_id.'-'.$field_id.'-field'],
+            $record_id,
+            $field_id,
+            $form_state['values'][$record_id.'-'.$field_id]
+          );
+        }
+      }
+  }
+  
+}
+
+function theme_tripal_bulk_loader_set_constants_form ($form) {
+  $output = '';
+  
+  $exposed_fields = unserialize($form['exposed_array']['#value']);
+  // need to put in the context of a node so we can use the has_exposed_fields function
+  if ($exposed_fields) {
+    $node->exposed_fields = $exposed_fields;
+  } else {
+    $node->exposed_fields = array();
+  }
+  
+  // Add draggable table for constant sets
+  if (tripal_bulk_loader_has_exposed_fields($node)) {
+    $i=1;
+    foreach (element_children($form['exposed_fields']['existing']) as $key) {
+      $element = &$form['exposed_fields']['existing'][$key];
+      $element['group']['#attributes']['class'] = 'weight-group';
+  
+      $row = array();
+      foreach ($exposed_fields as $exposed) {
+        if ($i==1) {
+          $header[] = $exposed['title'];
+        }
+        $k = $exposed['record_id'].'-'.$exposed['field_id'];
+        $row[] = drupal_render($element[$k]);
+      }
+      $row[] = drupal_render($element['delete']);
+      $row[] = drupal_render($element['group']) . drupal_render($element['id']);
+      $rows[] = array('data' => $row, 'class' => 'draggable');
+      $i++;
+    }
+    //drupal_add_tabledrag('mytable', 'order', 'sibling', 'weight-group');
+    $form['exposed_fields']['existing'] = array(
+      '#type' => 'markup',
+      '#value' => theme('table', $header, $rows, array('id' => 'mytable')) . '<br />'
+    );
+  }
+    
+  $output .= drupal_render($form);
+  return $output;
+}
+
+///////////////////////////////////////////////////////////
+// Set Constants Form (on Bulk Loader Node)
+///////////////////////////////////////////////////////////
+
+/**
+ * Edit a constant set (exposed fields in template)
+ *
+ * @param $form_state 
+ *   The current state of the form
+ * @param $node
+ *   The node to set constants for
+ * @param $group_id
+ *   The constant set to edit
+ *
+ * @return
+ *   A form array to be rendered by drupal_get_form()
+ */
+function tripal_bulk_loader_edit_constant_set_form ($form_state, $node, $group_id) {
+  $form = array();
+
+  $form['#redirect'] = 'node/'.$node->nid;
+  
+  $form['nid'] = array(
+    '#type' => 'hidden',
+    '#value' => $node->nid,
+  );
+
+  $form['group_id'] = array(
+    '#type' => 'hidden',
+    '#value' => $group_id,
+  );
+
+    
+  $form['explanation'] = array(
+    '#type' => 'item',
+    '#value' => t('The following fields are constants in the selected template that you need to set values for.')
+  );
+  
+  // Add textifelds for exposed fields of the current template
+  $exposed_fields = FALSE;
+  $indexes = array();
+  if (tripal_bulk_loader_has_exposed_fields($node)) {
+    foreach ($node->exposed_fields as $exposed_index) {
+      
+      $record_id = $exposed_index['record_id'];
+      $field_id = $exposed_index['field_id'];
+      $field = $node->template->template_array[$record_id]['fields'][$field_id];
+
+      if ($field['exposed']) {
+        $exposed_fields = TRUE;
+        $indexes[$record_id][] = $field_id;
+        
+        switch($field['type']) {
+          case 'table field':
+            $form[$record_id.'-'.$field_id] = array(
+              '#type' => 'textfield',
+              '#title' => t($field['title']),
+              '#description' => t($field['exposed_description']),
+              '#default_value' => (isset($node->constants[$group_id][$record_id][$field_id]['value'])) ? $node->constants[$group_id][$record_id][$field_id]['value'] : $field['constant value'],
+            );
+          break;
+          case 'constant':
+            $form[$record_id.'-'.$field_id] = array(
+              '#type' => 'textfield',
+              '#title' => t($field['title']),
+              '#description' => t('Enter the case-sensitive value of this constant for your spreadsheet'),
+              '#default_value' => (isset($node->constants[$group_id][$record_id][$field_id]['value'])) ? $node->constants[$group_id][$record_id][$field_id]['value'] : $field['constant value'],
+            );
+          break;
+        }
+        
+        $form[$record_id.'-'.$field_id.'-table'] = array(
+          '#type' => 'hidden',
+          '#value' => $record['table'],
+        );
+        $form[$record_id.'-'.$field_id.'-field'] = array(
+          '#type' => 'hidden',
+          '#value' => $field['field'],
+        );
+        $form[$record_id.'-'.$field_id.'-type'] = array(
+          '#type' => 'hidden',
+          '#value' => $field['type'],
+        );
+      }
+
+    }
+  }
+  $form['template'] = array(
+    '#type' => 'hidden',
+    '#value' => serialize($node->template->template_array)
+  );
+  
+  $form['indexes'] = array(
+    '#type' => 'hidden',
+    '#value' => serialize($indexes),
+  );
+  
+  $form['save'] = array(
+    '#type' => 'submit',
+    '#value' => 'Save',
+  );
+  
+  $form['cancel'] = array(
+    '#type' => 'submit',
+    '#value' => 'Cancel',
+  );
+    
+  return $form;
+}
+
+/**
+ * Edit constants in the current constant set
+ */
+function tripal_bulk_loader_edit_constant_set_form_submit ($form, $form_state) {
+  
+  // Update constants
+  $template = unserialize($form_state['values']['template']);
+  $indexes = unserialize($form_state['values']['indexes']);
+  
+  $op = $form_state['values'][ $form_state['clicked_button']['#name'] ];
+  if (strcmp('Save',$op) == 0) {
+
+    foreach ($indexes as $record_id => $array) {
+      foreach($array as $field_id) {	
+        tripal_bulk_loader_update_constant(
+          $form_state['values']['nid'], 
+          $form_state['values']['group_id'],
+          $form_state['values'][$record_id.'-'.$field_id.'-table'], 
+          $form_state['values'][$record_id.'-'.$field_id.'-field'],
+          $record_id,
+          $field_id,
+          $form_state['values'][$record_id.'-'.$field_id]
+        );
+      }
+    }
+    drupal_set_message('The constant set was successfully updated.');
+      
+  }
+  
+}
+
+/**
+ * Delete a constant set (exposed fields in template)
+ *
+ * @param $form_state 
+ *   The current state of the form
+ * @param $node
+ *   The node to set constants for
+ * @param $group_id
+ *   The constant set to delete
+ *
+ * @return
+ *   A form array to be rendered by drupal_get_form()
+ */
+function tripal_bulk_loader_delete_constant_set_form ($form_state, $node, $group_id) {
+  $form = array();
+
+  $form['#redirect'] = 'node/'.$node->nid;
+	
+	$form['nid'] = array(
+		'#type' => 'value',
+		'#value' => $node->nid,
+	);
+
+  $form['group_id'] = array(
+    '#type' => 'hidden',
+    '#value' => $group_id,
+  );
+   
+	return confirm_form($form,
+    	t('Are you sure you want to delete this constant set?'),
+    	'node/'.$node->nid,
+    	t('This action cannot be undone.'),
+    	t('Delete'),
+    	t('Cancel')
+  );
+  
+}
+
+/**
+ * Delete the current constant set
+ */
+function tripal_bulk_loader_delete_constant_set_form_submit ($form, $form_state) {
+
+  $group_id = $form_state['values']['group_id'];
+  $nid = $form_state['values']['nid'];
+  if ($nid && $form_state['values']['confirm']) {
+    db_query("DELETE FROM {tripal_bulk_loader_constants} WHERE nid=%d AND group_id=%d",$nid, $group_id);
+    drupal_set_message('Constant set successfully deleted.');
+  }
+        
+}

base/tripal_bulk_loader/tripal_bulk_loader.install

@@ -114,26 +114,32 @@ function tripal_bulk_loader_schema() {
         'type' => 'int',
         'unsigned' => TRUE,
         'not null' => TRUE,
-			),
-			'chado_table' => array(
-			  'type' => 'varchar',
-			  'not null' => TRUE,
-			),
-			'chado_field' => array(
-			  'type' => 'varchar',
-			  'not null' => TRUE,
-			),
-			'record_id' => array(
-			  'type' => 'int',
-			  'not null' => TRUE
-			),
-			'field_id' => array(
-			  'type' => 'int',
-			  'not null' => TRUE
-			),
-			'value' => array(
-			  'type' => 'text',
-			),
+      ),
+      'group_id' => array(
+        'type' => 'int',
+        'unsigned' => TRUE,
+        'not null' => TRUE,
+        'default' => 0
+      ),
+      'chado_table' => array(
+        'type' => 'varchar',
+        'not null' => TRUE,
+      ),
+      'chado_field' => array(
+        'type' => 'varchar',
+        'not null' => TRUE,
+      ),
+      'record_id' => array(
+        'type' => 'int',
+        'not null' => TRUE
+      ),
+      'field_id' => array(
+        'type' => 'int',
+        'not null' => TRUE
+      ),
+      'value' => array(
+        'type' => 'text',
+      ),
 	  ),
 	  'primary key' => array('constant_id'),
 	);
@@ -154,4 +160,28 @@ function tripal_bulk_loader_update_6150 () {
   
   return $ret;
   
+}
+
+/**
+ * Update schema for version 6.x-0.3.1b-1.5
+ * - Add the tripal_bulk_loader_constants.group_id column 
+ *   to allow multiple sets of constants per job
+ */
+function tripal_bulk_loader_update_6151 () {
+  $ret = array();
+  
+  $schema = tripal_bulk_loader_schema();
+  db_add_field(
+    $ret, 
+    'tripal_bulk_loader_constants', 
+    'group_id', 
+    array(
+        'type' => 'int',
+        'unsigned' => TRUE,
+        'not null' => TRUE,
+        'default' => 0
+      )
+    );
+
+  return $ret;  
 }

base/tripal_bulk_loader/tripal_bulk_loader.loader.inc

@@ -110,23 +110,13 @@ function tripal_bulk_loader_load_data($nid) {
   
   // set the status of the job (in the node not the tripal jobs)
   db_query("UPDATE tripal_bulk_loader SET job_status='%s' WHERE nid=%d",'Loading...', $nid);
-  
-  print "Memory Usage (Start): ".number_format((memory_get_usage() * 0.000000953674316), 5, '.', ',') . " Mb\n";
+
   
   $node = node_load($nid);
   print "Template: ".$node->template->name." (".$node->template_id.")\n";
-  print "File: ".$node->file."\n";
-  
-  if ($node->constants) {
-    print "Constants:\n";
-    foreach ($node->constants as $record_id => $record) {
-      foreach ($record as $field_id => $field) {
-        print "\t- ".$field['chado_table'].'.'.$field['chado_field'].' = '.$field['value']."\n";
-      }
-    }
-  }
-  
-  print "\nMemory Usage (After Node Load): ".number_format((memory_get_usage() * 0.000000953674316), 5, '.', ',') . " Mb\n";
+
+  $total_lines = trim(`wc --lines < $node->file`);
+  print "File: ".$node->file." (".$total_lines." lines)\n";
   
   // Prep Work ==================================================================================
   $loaded_without_errors = TRUE;
@@ -180,86 +170,101 @@ function tripal_bulk_loader_load_data($nid) {
     //watchdog('T_bulk_loader','2)'.$record_array['record_id'].':<pre>'.print_r($default_data[$priority], TRUE).'</pre>', array(), WATCHDOG_NOTICE);
   } //end of foreach record
 
-  // Add constants
-  if ($node->constants) {
-    foreach ($node->constants as $priority => $record) {
-      foreach ($record as $field_id => $field) {
-        if ($default_data[$priority]['table'] == $field['chado_table']) {
-          if (isset($default_data[$priority]['values_array'][$field['chado_field']])) {
-            if (isset($field2column[$priority][$field['chado_field']])) {
-              $field2column[$priority][$field['chado_field']] = $field['value'];
+  ///////////////////////////////////////////////
+  // For each set of constants
+  ///////////////////////////////////////////////
+  $original_default_data = $default_data;
+  $group_index = 0;
+  $total_num_groups = sizeof($node->constants);
+  foreach ($node->constants as $group_id => $set) {
+    // revert default data array for next set of constants
+    $default_data = $original_default_data;
+    $group_index++;
+    
+    // Add constants
+    if (!empty($set)) {
+      print "Constants:\n";
+      foreach ($set as $priority => $record) {
+        foreach ($record as $field_id => $field) {
+        
+          print "\t- ".$field['chado_table'].'.'.$field['chado_field'].' = '.$field['value']."\n";
+          
+          if ($default_data[$priority]['table'] == $field['chado_table']) {
+            if (isset($default_data[$priority]['values_array'][$field['chado_field']])) {
+              if (isset($field2column[$priority][$field['chado_field']])) {
+                $field2column[$priority][$field['chado_field']] = $field['value'];
+              } else {
+                $default_data[$priority]['values_array'][$field['chado_field']] = $field['value'];
+              }
             } else {
-              $default_data[$priority]['values_array'][$field['chado_field']] = $field['value'];
+              print "ERROR: Template has changed after constants were assigned!\n";
+              watchdog('T_bulk_loader','Template has changed after constants were assigned', array(), WATCHDOG_NOTICE);
+              exit(1);
             }
           } else {
             print "ERROR: Template has changed after constants were assigned!\n";
             watchdog('T_bulk_loader','Template has changed after constants were assigned', array(), WATCHDOG_NOTICE);
             exit(1);
           }
-        } else {
-          print "ERROR: Template has changed after constants were assigned!\n";
-          watchdog('T_bulk_loader','Template has changed after constants were assigned', array(), WATCHDOG_NOTICE);
-          exit(1);
         }
       }
     }
-  }
-  
-  print "Memory Usage (end of prep work): ".number_format((memory_get_usage() * 0.000000953674316), 5, '.', ',') . " Mb\n";
-  
-  //print "Default Data:".print_r($default_data,TRUE)."\n";
-  //watchdog('T_bulk_loader','Default Data:<pre>'.print_r($default_data, TRUE).'</pre>', array(), WATCHDOG_NOTICE);
-  
-  //print "\nDefault Values Array: ".print_r($default_data, TRUE)."\n";
-  //print "\nField to Column Mapping: ".print_r($field2column, TRUE)."\n";
-  
-  // Parse File adding records as we go ========================================================
-  $file_handle = fopen($node->file, 'r');
-  if (preg_match('/(t|true|1)/', $node->file_has_header)) { fgets($file_handle, 4096); }
-  $num_records = 0;
-  $num_lines = 0;
-  $num_errors = 0;
-  $total_lines = trim(`wc --lines < $node->file`);
-  $interval = intval($total_lines * 0.01);
-  if($interval == 0){ $interval = 1; }
-  while (!feof($file_handle)) {
-
-    // Clear variables
-    // Was added to fix memory leak
-    unset($line);                     unset($raw_line);
-    unset($data);                     unset($data_keys);
-    unset($priority);                 unset($sql);
-    unset($result);                   
     
-    $raw_line = fgets($file_handle, 4096);
-    $raw_line = trim($raw_line);
-    if (empty($raw_line)) { continue; } // skips blank lines
-    $line = explode("\t", $raw_line);
-    $num_lines++;
+    //print "Default Data:".print_r($default_data,TRUE)."\n";
+    //watchdog('T_bulk_loader','Default Data:<pre>'.print_r($default_data, TRUE).'</pre>', array(), WATCHDOG_NOTICE);
     
-    // update the job status every 1% features
-    if($node->job_id and $num_lines % $interval == 0){
-      print "Updating progress of ".$node->job_id." to ".($num_lines/$interval) ."%\n";
-       tripal_job_set_progress($node->job_id,($num_lines/$interval));
-    }    
-
-    $data = $default_data;
-
-    $data_keys = array_keys($data); 
-    foreach ($data_keys as $priority) {
-      $status = process_data_array_for_line($priority, $data, $default_data, $field2column, $record2priority, $line, $nid);
-      if (!$status ) { $loaded_without_errors = FALSE; }
-    } // end of foreach table in default data array
-
-    // determine memory increase
-    $line_mem_increase = memory_get_usage() - $memory;
-    if ($num_lines != 1) { 
-      $increased_mem = $increased_mem + $line_mem_increase;
-    }
-    $memory = memory_get_usage();
+    //print "\nDefault Values Array: ".print_r($default_data, TRUE)."\n";
+    //print "\nField to Column Mapping: ".print_r($field2column, TRUE)."\n";
+    
+    // Parse File adding records as we go ========================================================
+    $file_handle = fopen($node->file, 'r');
+    if (preg_match('/(t|true|1)/', $node->file_has_header)) { fgets($file_handle, 4096); }
+    $num_records = 0;
+    $num_lines = 0;
+    $num_errors = 0;
+    $interval = intval($total_lines * 0.10);
+    if($interval == 0){ $interval = 1; }
+    while (!feof($file_handle)) {
+  
+      // Clear variables
+      // Was added to fix memory leak
+      unset($line);                     unset($raw_line);
+      unset($data);                     unset($data_keys);
+      unset($priority);                 unset($sql);
+      unset($result);                   
+      
+      $raw_line = fgets($file_handle, 4096);
+      $raw_line = trim($raw_line);
+      if (empty($raw_line)) { continue; } // skips blank lines
+      $line = explode("\t", $raw_line);
+      $num_lines++;
+      
+      // update the job status every 10% of lines processed for the current group
+      if($node->job_id and $num_lines % $interval == 0){
+        // percentage of lines processed for the current group
+        $group_progress = round(($num_lines/$total_lines)*100);
+        // percentage of lines processed for all groups
+        $job_progress = round(($group_index / $total_num_groups) * $group_progress);
+        
+          print "\nProgress Update:\n"
+            ."\t- ".$num_lines." lines have been processed for the current constant set.\n"
+            ."\t- ".$group_progress."% of the lines in the file have been processed for the current constant set.\n"
+            ."\t- ".$job_progress."% of the current job has been completed.\n";
+            
+        tripal_job_set_progress($node->job_id,$job_progress);
+      }    
+  
+      $data = $default_data;
+  
+      $data_keys = array_keys($data); 
+      foreach ($data_keys as $priority) {
+        $status = process_data_array_for_line($priority, $data, $default_data, $field2column, $record2priority, $line, $nid, $num_lines);
+        if (!$status ) { $loaded_without_errors = FALSE; }
+      } // end of foreach table in default data array
+      
+    } //end of foreach line of file
+  } //end of foreach constant set
     
-  } //end of foreach line of file
-  
   // check that data was inserted and update job_status
   $sql = 'SELECT count(*) as num_tables FROM {tripal_bulk_loader_inserted} WHERE nid=%d GROUP BY nid';
   $result = db_fetch_object(db_query($sql, $nid));
@@ -271,11 +276,6 @@ function tripal_bulk_loader_load_data($nid) {
   // set the status of the job (in the node not the tripal jobs)
   if ($loaded_without_errors) { $status = 'Loading Completed Successfully'; } else { $status = 'Errors Encountered'; }
   db_query("UPDATE tripal_bulk_loader SET job_status='%s' WHERE nid=%d",$status, $nid);
-  
-  $avg_line_increase = ( $increased_mem / $num_lines) * 0.0078125;
-  print "Average Increase in Memory per Line: ".number_format($avg_line_increase, 5, '.', ',') . " Kb\n";
-  print "Peak Memory Usage: ".number_format((memory_get_peak_usage() * 0.000000953674316), 5, '.', ',') . " Mb\n";
-  print "End Memory Usage: ".number_format((memory_get_usage() * 0.000000953674316), 5, '.', ',') . " Mb\n";
 
 }
 
@@ -283,7 +283,7 @@ function tripal_bulk_loader_load_data($nid) {
  * 
  *
  */
-function process_data_array_for_line ($priority, &$data, &$default_data, $field2column, $record2priority, $line, $nid) {
+function process_data_array_for_line ($priority, &$data, &$default_data, $field2column, $record2priority, $line, $nid, $line_num) {
   $table_data = $data[$priority];
 
   $no_errors = TRUE;
@@ -297,21 +297,21 @@ function process_data_array_for_line ($priority, &$data, &$default_data, $field2
   if ($table_data['need_further_processing']) {
     $values = tripal_bulk_loader_add_spreadsheetdata_to_values ($values, $line, $field2column[$priority]);
     if (!$values) {
-      watchdog('T_bulk_loader','Spreadsheet Added:'.print_r($values, TRUE), array(), WATCHDOG_NOTICE);
+      watchdog('T_bulk_loader','Line '.$line_num.' Spreadsheet Added:'.print_r($values, TRUE), array(), WATCHDOG_NOTICE);
     }
     
     $values = tripal_bulk_loader_add_foreignkey_to_values($values, $data, $record2priority);
     if (!$values) {
-      watchdog('T_bulk_loader','FK Added:<pre>'.print_r($values, TRUE).print_r($data[$priority],TRUE).'</pre>', array(), WATCHDOG_NOTICE);
+      watchdog('T_bulk_loader','Line '.$line_num.' FK Added:<pre>'.print_r($values, TRUE).print_r($data[$priority],TRUE).'</pre>', array(), WATCHDOG_NOTICE);
     }
   }
   $values = tripal_bulk_loader_regex_tranform_values($values, $table_data, $line);
   if (!$values) {
-    watchdog('T_bulk_loader','Regex:<pre>'.print_r($values, TRUE).print_r($table_data, TRUE).'</pre>'.'</pre>', array(), WATCHDOG_NOTICE);
+    watchdog('T_bulk_loader','Line '.$line_num.' Regex:<pre>'.print_r($values, TRUE).print_r($table_data, TRUE).'</pre>'.'</pre>', array(), WATCHDOG_NOTICE);
   }
 
   if (!$values) {
-    $msg = $table_data['record_id'].' ('.$table_data['mode'].') Aborted due to error in previous record. Values of current record:'.print_r($table_data['values_array'],TRUE);
+    $msg = 'Line '.$line_num.' '.$table_data['record_id'].' ('.$table_data['mode'].') Aborted due to error in previous record. Values of current record:'.print_r($table_data['values_array'],TRUE);
     watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING); 
     print "ERROR: ".$msg."\n";
     $data[$priority]['error'] = TRUE;
@@ -326,7 +326,7 @@ function process_data_array_for_line ($priority, &$data, &$default_data, $field2
       // a field is considered missing if it cannot be null and there is no default
       // value for it or it is of type 'serial'
       if($def['not null'] == 1 and !array_key_exists($field,$insert_values) and !isset($def['default']) and strcmp($def['type'],serial)!=0){
-         $msg = $table_data['record_id'].' ('.$table_data['mode'].') Missing Database Required Value: '.$table.'.'.$field;
+         $msg = 'Line '.$line_num.' '.$table_data['record_id'].' ('.$table_data['mode'].') Missing Database Required Value: '.$table.'.'.$field;
          watchdog('T_bulk_loader', $msg, array(), WATCHDOG_NOTICE); 
          $data[$priority]['error'] = TRUE;
       }
@@ -337,7 +337,7 @@ function process_data_array_for_line ($priority, &$data, &$default_data, $field2
   foreach ($table_data['required'] as $field => $required) {
     if ($required) {
       if (!isset($values[$field])) {
-        $msg = $table_data['record_id'].' ('.$table_data['mode'].') Missing Template Required Value: '.$table.'.'.$field;
+        $msg = 'Line '.$line_num.' '.$table_data['record_id'].' ('.$table_data['mode'].') Missing Template Required Value: '.$table.'.'.$field;
         watchdog('T_bulk_loader', $msg, array(), WATCHDOG_NOTICE); 
         $data[$priority]['error'] = TRUE;  
       }
@@ -381,7 +381,7 @@ function process_data_array_for_line ($priority, &$data, &$default_data, $field2
     //watchdog('T_bulk_loader',$header.': Inserting:'.print_r($values, TRUE), array(), WATCHDOG_NOTICE);
     $record = tripal_core_chado_insert($table, $values);
     if (!$record) {
-      $msg = $table_data['record_id'].' ('.$table_data['mode'].') Unable to insert record into '.$table.' where values:'.print_r($values,TRUE);
+      $msg = 'Line '.$line_num.' '.$table_data['record_id'].' ('.$table_data['mode'].') Unable to insert record into '.$table.' where values:'.print_r($values,TRUE);
       watchdog('T_bulk_loader', $msg, array(), WATCHDOG_ERROR); 
       print "ERROR: ".$msg."\n";  
       $data[$priority]['error'] = TRUE;
@@ -424,7 +424,7 @@ function process_data_array_for_line ($priority, &$data, &$default_data, $field2
     $exists = tripal_core_chado_select($table, array_keys($table_desc['fields']), $values, array('has_record'=>TRUE));
     if (!$exists) {
       // No record on select
-      $msg = $table_data['record_id'].' ('.$table_data['mode'].') No Matching record in '.$table.' where values:'.print_r($values,TRUE);
+      $msg = 'Line '.$line_num.' '.$table_data['record_id'].' ('.$table_data['mode'].') No Matching record in '.$table.' where values:'.print_r($values,TRUE);
       watchdog('T_bulk_loader', $msg, array(), WATCHDOG_WARNING); 
       $data[$priority]['error'] = TRUE;  
     }

base/tripal_bulk_loader/tripal_bulk_loader.module

@@ -1,6 +1,7 @@
 <?php
 
 include('tripal_bulk_loader.loader.inc');
+include('tripal_bulk_loader.constants.inc');
 
 /**
  * Implements hook_init
@@ -25,6 +26,25 @@ function tripal_bulk_loader_menu() {
      'access arguments' => array('access tripal_bulk_loader'),
      'type' => MENU_NORMAL_ITEM,
    );
+   // Bulk Loading Job Node
+   $items['node/%node/constants/%/edit'] = array(
+      'title' => 'Edit Constant Set',
+      'description' => 'Edit a group of constants associated with the current bulk loader',
+    	'page callback' => 'drupal_get_form',
+    	'page arguments' => array('tripal_bulk_loader_edit_constant_set_form',1,3),
+      'access arguments' => array('administer site configuration'),
+      'type' => MENU_CALLBACK,   
+		  'file' => 'tripal_bulk_loader.constants.inc',
+   );
+   $items['node/%node/constants/%/delete'] = array(
+      'title' => 'Delete Constant Set',
+      'description' => 'Delete a group of constants associated with the current bulk loader',
+    	'page callback' => 'drupal_get_form',
+    	'page arguments' => array('tripal_bulk_loader_delete_constant_set_form',1,3),
+      'access arguments' => array('administer site configuration'),
+      'type' => MENU_CALLBACK,   
+		  'file' => 'tripal_bulk_loader.constants.inc',
+   );
    // Admin page to create the template
    $items['admin/tripal/tripal_bulk_loader_template'] = array(
       'title' => 'Bulk Loader Template',
@@ -126,11 +146,15 @@ function tripal_bulk_loader_menu() {
   return $items;
 }
 
+
 /**
  * Implements hook_theme
  */
 function tripal_bulk_loader_theme() {
   return array(
+    'tripal_bulk_loader_set_constants_form' => array(
+      'arguments' => array('form' => NULL),
+    ),
     'tripal_bulk_loader_template' => array(
     	'arguments'=> array('template_id' => NULL),
     	'template' => 'tripal_bulk_loader_template'
@@ -333,12 +357,35 @@ function tripal_bulk_loader_load($node){
     $r->num_inserted = sizeof(preg_split('/,/',$r->ids_inserted)); 
     $node->inserted_records->{$r->table_inserted_into} = $r;
   }
+
+  // Add exposed field list
+  $node->exposed_fields = array();
+  if ($template) {
+    foreach ($template as $record_id => $record) {
+      foreach ($record['fields'] as $field_id => $field) {
+        if ($field['exposed']) {
+          $node->exposed_fields[] = array(
+            'record_id' => $record_id,
+            'field_id' => $field_id,
+            'title' => $field['title'],
+          );
+        }
+      }
+    }
+    
+    if (empty($node->exposed_fields)) {
+      $node->exposed_fields[] = array();
+    }
+  }
+
   
   // Add constants
-  $sql = 'SELECT * FROM {tripal_bulk_loader_constants} WHERE nid=%d ORDER BY record_id, field_id';
+  $sql = 'SELECT * FROM {tripal_bulk_loader_constants} WHERE nid=%d ORDER BY group_id, record_id, field_id';
   $resource = db_query($sql,$node->nid);
   while ($r = db_fetch_object($resource)) {
-    $node->constants[$r->record_id][$r->field_id] = array(
+    $node->constants[$r->group_id][$r->record_id][$r->field_id] = array(
+      'constant_id' => $r->constant_id,
+      'group_id' => $r->group_id,
       'chado_table'=>$r->chado_table, 
       'chado_field'=>$r->chado_field, 
       'record_id'=>$r->record_id, 
@@ -346,6 +393,9 @@ function tripal_bulk_loader_load($node){
       'value'=>$r->value
     );
   }
+  if (!$node->constants) {
+    $node->constants[] = array();
+  }
   
 	return $node;
 }
@@ -404,228 +454,6 @@ function tripal_bulk_loader_update ($node) {
   
 }
 
-///////////////////////////////////////////////////////////
-// Set Constants Form
-///////////////////////////////////////////////////////////
-
-/**
- * Set constants (exposed fields in template)
- *
- * @param $form_state 
- *   The current state of the form
- * @param $node
- *   The node to set constants for
- *
- * @return
- *   A form array to be rendered by drupal_get_form()
- */
-function tripal_bulk_loader_set_constants_form ($form_state, $node) {
-  $form = array();
-  
-  $form['nid'] = array(
-    '#type' => 'hidden',
-    '#value' => $node->nid
-  );
-  
-  $form['exposed_fields'] = array(
-    '#type' => 'fieldset',
-    '#title' => t('Constant Values'),
-    '#collapsible' => TRUE,
-    '#collapsed' => ($node->template_id) ? FALSE : TRUE,
-    '#prefix' => '<div id="set-constants">',
-    '#suffix' => '</div>',
-  );
-  
-  $form['exposed_fields']['explanation'] = array(
-    '#type' => 'item',
-    '#value' => t('The following fields are constants in the selected template that you need to set values for.')
-  );
-  
-  // Add textifelds for exposed fields of the current template
-  $exposed_fields = FALSE;
-  $indexes = array();
-  if ($node->template_id) {
-    if (isset($node->template)) {
-      foreach ($node->template->template_array as $record_id => $record) {
-        foreach ($record['fields'] as $field_id => $field) {
-          if ($field['exposed']) {
-            $exposed_fields = TRUE;
-            $indexes[$record_id][] = $field_id;
-            
-            switch($field['type']) {
-              case 'table field':
-                $form['exposed_fields'][$record_id.'-'.$field_id] = array(
-                  '#type' => 'textfield',
-                  '#title' => t($field['title']),
-                  '#description' => t($field['exposed_description']),
-                  '#default_value' => (isset($node->constants[$record_id][$field_id]['value'])) ? $node->constants[$record_id][$field_id]['value'] : $field['constant value'],
-                );
-              break;
-              case 'constant':
-                $form['exposed_fields'][$record_id.'-'.$field_id] = array(
-                  '#type' => 'textfield',
-                  '#title' => t($field['title']),
-                  '#description' => t('Enter the case-sensitive value of this constant for your spreadsheet'),
-                  '#default_value' => (isset($node->constants[$record_id][$field_id]['value'])) ? $node->constants[$record_id][$field_id]['value'] : $field['constant value'],
-                );
-              break;
-            }
-            
-            $form['exposed_fields'][$record_id.'-'.$field_id.'-table'] = array(
-              '#type' => 'hidden',
-              '#value' => $record['table'],
-            );
-            $form['exposed_fields'][$record_id.'-'.$field_id.'-field'] = array(
-              '#type' => 'hidden',
-              '#value' => $field['field'],
-            );
-            $form['exposed_fields'][$record_id.'-'.$field_id.'-type'] = array(
-              '#type' => 'hidden',
-              '#value' => $field['type'],
-            );
-          }
-        }
-      }
-    }
-  }
-  $form['template'] = array(
-    '#type' => 'hidden',
-    '#value' => serialize($node->template->template_array)
-  );
-  
-  $form['exposed_fields']['indexes'] = array(
-    '#type' => 'hidden',
-    '#value' => serialize($indexes),
-  );
-  
-  if (!$exposed_fields) {
-    $form['exposed_fields']['explanation'] = array(
-      '#type' => 'item',
-      '#value' => t('There are no exposed fields for this template.')
-    );    
-  }
-  
-  $form['exposed_fields']['submit'] = array(
-    '#type' => 'submit',
-    '#value' => t('Save Changes')
-  );
-  
-  return $form;
-}
-
-/**
- * Validate that the values entered exist in the database
- * if indicated in hte template array
- */
-function tripal_bulk_loader_set_constants_form_validate ($form_state, $form) {
-
-  $template = unserialize($form_state['template']['#value']);
-  $indexes = unserialize($form_state['exposed_fields']['indexes']['#value']);
-  
-  foreach ($indexes as $record_id => $array) {
-    foreach($array as $field_id) {
-      if ($template[$record_id]['fields'][$field_id]['exposed_validate']) {
-        $result = db_fetch_object(db_query(
-          "SELECT 1 as valid FROM %s WHERE %s='%s'",
-          $template[$record_id]['table'],
-          $template[$record_id]['fields'][$field_id]['field'],
-          $form_state['exposed_fields'][$record_id.'-'.$field_id]['#value']
-        ));
-
-        if (!$result->valid) {
-          $msg = 'A '.$form_state['exposed_fields'][$record_id.'-'.$field_id]['#title'].' of "'.$form_state['exposed_fields'][$record_id.'-'.$field_id]['#value'].'" must already exist!';
-          form_set_error($record_id.'-'.$field_id, $msg);
-        } else {
-          drupal_set_message('Confirmed a '.$form_state['exposed_fields'][$record_id.'-'.$field_id]['#title'].' of "'.$form_state['exposed_fields'][$record_id.'-'.$field_id]['#value'].'" already exists.');
-        }
-      }
-    }
-  }
-  
-}
-
-/**
- * Insert/update the constants associated with this node
- */
-function tripal_bulk_loader_set_constants_form_submit ($form, $form_state) {
-
-	// Insert/Update constants
-  $template = unserialize($form_state['values']['template']);
-  $indexes = unserialize($form_state['values']['indexes']);
-  
-  foreach ($indexes as $record_id => $array) {
-    foreach($array as $field_id) {	
-      tripal_bulk_loader_update_constant(
-        $form_state['values'][nid], 
-        $form_state['values'][$record_id.'-'.$field_id.'-table'], 
-        $form_state['values'][$record_id.'-'.$field_id.'-field'],
-        $record_id,
-        $field_id,
-        $form_state['values'][$record_id.'-'.$field_id]
-        );
-    }
-  }
-  
-}
-
-/**
- * Inserts/Updates a tripal bulk loading job constant
- *
- * @param $nid
- *   The node ID of the the tripal bulk loading job the constant is associated with
- * @param $table
- *   The chado table the constant is associated with
- * @param $field
- *   The chado field the constant is associated with
- * @param $record_id
- *   The index in the template array for this record
- * @param $field_id
- *   The index in the template array for this field
- *
- * NOTE: $template_array[$record_id]['table'] = $table and $template_array[$record_id]['fields'][$field_id]['field'] = $field
- *   both are included as a means of double-checking the constant still is still in thesame place in the template array.
- *   For example, that the template was not edited and the records moved around after the job was submitted but before it was run.
- *
- * @return
- *   On success it returns the object (with primary key if inserted);
- *   on failure it returns FALSE
- */
-function tripal_bulk_loader_update_constant ($nid, $table, $field, $record_id, $field_id, $value) {
-  
-  $record = array(
-    'nid'=>$nid, 
-    'chado_table'=>$table, 
-    'chado_field'=>$field, 
-    'record_id'=>$record_id, 
-    'field_id'=>$field_id, 
-    'value'=>$value
-  );
-  
-  // Check to see if already exists
-  $exists = db_fetch_object(db_query(
-    "SELECT constant_id FROM {tripal_bulk_loader_constants} WHERE nid=%d AND record_id=%d AND field_id=%d",
-    $record['nid'],
-    $record['record_id'],
-    $record['field_id']
-  ));
-  if ($exists->constant_id) {
-    $record['constant_id'] = $exists->constant_id;
-    $status = drupal_write_record('tripal_bulk_loader_constants',$record,'constant_id');
-    if ($status) {
-      return $record;
-    } else {
-      return FALSE;
-    }
-  } else {
-    $status = drupal_write_record('tripal_bulk_loader_constants',$record);
-    if ($status) {
-      return $record;
-    } else {
-      return FALSE;
-    }
-  }
-}
-
 ///////////////////////////////////////////////////////////
 
 /**