Added progress indicator to the TripalJob class:

tripal/api/tripal.collections.api.inc

@@ -182,15 +182,7 @@ function tripal_get_collection($values) {
  * @ingroup tripal_data_collections_api
  */
 function tripal_create_collection_files($collection_id, TripalJob $job = NULL) {
-  if ($job) {
-    $job->setProgress(0);
-  }
-
   $collection = new TripalEntityCollection();
   $collection->load($collection_id);
-  $collection->writeAll();
-
-  if ($job) {
-    $job->setProgress(100);
-  }
+  $collection->writeAll($job);
 }

tripal/includes/TripalEntityCollection.inc

@@ -624,11 +624,14 @@ class TripalEntityCollection {
   /**
    * Writes the collection to all file downloadable formats.
    *
+   * @param $job
+   *    If this function is run as a Tripal Job then this argument can be
+   *    set to the Tripaljob object for keeping track of progress.
    * @throws Exception
    */
-  public function writeAll() {
+  public function writeAll(TripalJob $job = NULL) {
     foreach ($this->downloaders as $class_name => $label) {
-      $this->write($class_name);
+      $this->write($class_name, $job);
     }
   }
 
@@ -673,10 +676,12 @@ class TripalEntityCollection {
    * @param formatter
    *   The name of the formatter class to use (e.g. TripalTabDownloader). The
    *   formatter must be compatible with the data collection.
-   *
+   * @param $job
+   *    If this function is run as a Tripal Job then this argument can be
+   *    set to the Tripaljob object for keeping track of progress.
    * @throws Exception
    */
-  public function write($formatter) {
+  public function write($formatter, TripalJob $job = NULL) {
     if (!$this->isFormatterCompatible($formatter)) {
       throw new Exception(t('The formatter, "@formatter", is not compatible with this data collection.', array('@formatter' => $formatter)));
 
@@ -689,7 +694,7 @@ class TripalEntityCollection {
     $outfile = $this->getOutfile($formatter);
 
     $downloader = new $formatter($this->collection_id, $outfile);
-    $downloader->write();
+    $downloader->write($job);
 
   }
 

tripal/includes/TripalFieldDownloaders/TripalFieldDownloader.inc

@@ -220,8 +220,13 @@ abstract class TripalFieldDownloader {
 
   /**
    * Creates the downloadable file.
+   *
+   * @param $job
+   *    If this function is run as a Tripal Job then this argument can be
+   *    set to the Tripaljob object for keeping track of progress.
    */
-  public function write() {
+  public function write(TripalJob $job = NULL) {
+
     $user = user_load($this->collection->uid);
 
     $fh = fopen(drupal_realpath($this->outfile), "w");
@@ -237,7 +242,17 @@ abstract class TripalFieldDownloader {
       }
     }
 
+    // Count the total number of entities
+    $total_entities = 0;
     $bundle_collections = $this->collection_bundles;
+    foreach ($bundle_collections as $bundle_collection) {
+      $total_entities += count($bundle_collection->ids);
+    }
+    if ($job) {
+      $job->setTotalItems($total_entities);
+    }
+
+    $num_handled = 0;
     foreach ($bundle_collections as $bundle_collection) {
       $collection_bundle_id = $bundle_collection->collection_bundle_id;
       $bundle_name = $bundle_collection->bundle_name;
@@ -246,6 +261,10 @@ abstract class TripalFieldDownloader {
       $site_id = $bundle_collection->site_id;
 
       foreach ($entity_ids as $entity_id) {
+        $num_handled++;
+        if ($job) {
+          $job->setItemsHandled($num_handled);
+        }
 
         // if we have a site_id then we need to get the entity from the
         // remote service. Otherwise create the entity from the local system.
@@ -306,6 +325,10 @@ abstract class TripalFieldDownloader {
         file_usage_add($file, 'tripal', 'data-collection', $fid);
       }
     }
+
+    if ($job) {
+      $job->setItemsHandled($num_handled);
+    }
   }
 
   /**

tripal/includes/TripalJob.inc

@@ -12,6 +12,35 @@ class TripalJob {
    */
   protected $job = NULL;
 
+
+  /**
+   * The number of items that this importer needs to process. A progress
+   * can be calculated by dividing the number of items process by this
+   * number.
+   */
+  private $total_items;
+
+  /**
+   * The number of items that have been handled so far.  This must never
+   * be below 0 and never exceed $total_items;
+   */
+  private $num_handled;
+
+  /**
+   * The interval when the job progress should be updated. Updating the job
+   * progress incurrs a database write which takes time and if it occurs to
+   * frequently can slow down the loader.  This should be a value between
+   * 0 and 100 to indicate a percent interval (e.g. 1 means update the
+   * progress every time the num_handled increases by 1%).
+   */
+  private $interval;
+
+  /**
+   * Each time the job progress is updated this variable gets set.  It is
+   * used to calculate if the $interval has passed for the next update.
+   */
+  private $prev_update;
+
   /**
    * Instantiates a new TripalJob object.
    *
@@ -403,6 +432,81 @@ class TripalJob {
       ->condition('job_id', $this->job->job_id)
       ->execute();
   }
+
+  /**
+   * Sets the total number if items to be processed.
+   *
+   * This should typically be called near the beginning of the loading process
+   * to indicate the number of items that must be processed.
+   *
+   * @param $total_items
+   *   The total number of items to process.
+   */
+  public function setTotalItems($total_items) {
+    $this->total_items = $total_items;
+  }
+
+  /**
+   * Adds to the count of the total number of items that have been handled.
+   *
+   * @param $num_handled
+   */
+  public function addItemsHandled($num_handled) {
+    $items_handled = $this->num_handled = $this->num_handled + $num_handled;
+    $this->setItemsHandled($items_handled);
+  }
+  /**
+   * Sets the number of items that have been processed.
+   *
+   * This should be called anytime the loader wants to indicate how many
+   * items have been processed.  The amount of progress will be
+   * calculated using this number.  If the amount of items handled exceeds
+   * the interval specified then the progress is reported to the user.  If
+   * this loader is associated with a job then the job progress is also updated.
+   *
+   * @param $total_handled
+   *   The total number of items that have been processed.
+   */
+  public function setItemsHandled($total_handled) {
+    // First set the number of items handled.
+    $this->num_handled = $total_handled;
+
+    if ($total_handled == 0) {
+      $memory = number_format(memory_get_usage());
+      print "Percent complete: 0%. Memory: " . $memory . " bytes.\r";
+      return;
+    }
+
+    // Now see if we need to report to the user the percent done.  A message
+    // will be printed on the command-line if the job is run there.
+    $percent = sprintf("%.2f", ($this->num_handled / $this->total_items) * 100);
+    $diff = $percent - $this->prev_update;
+
+    if ($diff >= $this->interval) {
+
+      $memory = number_format(memory_get_usage());
+      print "Percent complete: " . $percent . "%. Memory: " . $memory . " bytes.\r";
+      $this->prev_update = $diff;
+      $this->setProgress($percent);
+    }
+  }
+
+  /**
+   * Updates the percent interval when the job progress is updated.
+   *
+   * Updating the job
+   * progress incurrs a database write which takes time and if it occurs to
+   * frequently can slow down the loader.  This should be a value between
+   * 0 and 100 to indicate a percent interval (e.g. 1 means update the
+   * progress every time the num_handled increases by 1%).
+   *
+   * @param $interval
+   *   A number between 0 and 100.
+   */
+  public function setInterval($interval) {
+    $this->interval = $interval;
+  }
+
   /**
    * Retrieves the status of the job.
    */