tripal/tripal_bulk_loader/tripal_bulk_loader.module

<?php

/**
 * @file
 * The functions for the Tripal bulk loader.
 *
 * @defgroup tripal_bulk_loader Tripal Bulk Loader Module
 * @ingroup tripal_modules
 */

// Loader
include('includes/tripal_bulk_loader.loader.inc');

// Node
include('includes/tripal_bulk_loader.chado_node.inc');
include('includes/tripal_bulk_loader.constants.inc');

// Administration
include('includes/tripal_bulk_loader.admin.inc');
include('includes/tripal_bulk_loader.admin.templates.inc');

// API
include('api/tripal_bulk_loader.api.templates.inc');

/**
 * Implements hook_init
 * Used to add stylesheets and javascript files to the header
 *
 * @ingroup tripal_bulk_loader
 */
function tripal_bulk_loader_init() {
  // Add javascript and style sheet
  drupal_add_css(drupal_get_path('theme', 'tripal') . '/css/tripal_bulk_loader.css');
  drupal_add_js(drupal_get_path('theme', 'tripal') . '/js/tripal_bulk_loader.js');
}

/**
 * Implements hook_menu
 *
 * @ingroup tripal_bulk_loader
 */
function tripal_bulk_loader_menu() {
  $items = array();
  // Bulk Loading Job Node
  $items['node/%node/constants/%/edit'] = array(
    'title' => 'Edit Constant Set',
    'description' => 'Edit a group of constants associated with the current bulk loader',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_edit_constant_set_form', 1, 3),
    'access arguments' => array('administer tripal_bulk_loader'),
    'type' => MENU_CALLBACK,
  );
  $items['node/%node/constants/%/delete'] = array(
    'title' => 'Delete Constant Set',
    'description' => 'Delete a group of constants associated with the current bulk loader',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_delete_constant_set_form', 1, 3),
    'access arguments' => array('administer tripal_bulk_loader'),
    'type' => MENU_CALLBACK,
  );

  // Admin pages -----------------
  $items['admin/tripal/loaders/bulk'] = array(
    'title' => 'Bulk Loader',
    'description' => 'Templates for loading tab-delimited data',
    'page callback' => 'tripal_bulk_loader_admin_jobs_listing',
    'access arguments' => array('administer tripal_bulk_loader'),
    'type' => MENU_NORMAL_ITEM,
  );
  $items['admin/tripal/loaders/bulk/configure'] = array(
    'title' => 'Configure',
    'description' => 'Configuration of global options related to bulk loading jobs',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_configuration_form'),
    'access arguments' => array('administer tripal_bulk_loader'),
    'weight' => 8,
    'type' => MENU_LOCAL_TASK,
  );
  $items['admin/tripal/loaders/bulk/help'] = array(
    'title' => 'Help',
    'description' => "A description of the Tripal Bulk Loader module including a short description of it's usage.",
    'page callback' => 'tripal_bulk_loader_admin_manage_templates',
    'access arguments' => array('administer tripal_bulk_loader'),
    'weight' => 10,
    'type' => MENU_LOCAL_TASK,
  );

  // Enable View Callbacks
  $items['admin/tripal/loaders/bulk/views/jobs/enable'] = array(
    'title' => 'Enable Jobs Administrative View',
    'description' => 'Enable Jobs Administrative View',
    'page callback' => 'tripal_views_admin_enable_view',
    'page arguments' => array('tripal_bulk_loading_jobs','admin/tripal/loaders/bulk'),
    'access arguments' => array('administer tripal_bulk_loader'),
    'type' => MENU_CALLBACK,
  );
  $items['admin/tripal/loaders/bulk/views/templates/enable'] = array(
    'title' => 'Enable Templates Administrative View',
    'description' => 'Enable Templates Administrative View',
    'page callback' => 'tripal_views_admin_enable_view',
    'page arguments' => array('tripal_bulk_loader_templates', 'admin/tripal/loaders/bulk'),
    'access arguments' => array('administer tripal_bulk_loader'),
    'type' => MENU_CALLBACK,
  );

  // Create/Edit Template --------
  $items['admin/tripal/loaders/bulk/template/create'] = array(
    'title' => 'Create Template',
    'description' => 'Create loader template for loading tab-delimited data',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_modify_template_base_form', 'create'),
    'access arguments' => array('administer tripal_bulk_loader'),
    'weight' => -8,
    'type' => MENU_CALLBACK,
  );
  $items['admin/tripal/loaders/bulk/template/%tblid/edit'] = array(
    'title' => 'Edit Template',
    'description' => 'Edit loader template for loading tab-delimited data',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_modify_template_base_form', 'edit',5),
    'access arguments' => array('administer tripal_bulk_loader'),
    'type' => MENU_CALLBACK,
  );
  /**
  $items['admin/tripal/loaders/bulk/template/edit'] = array(
    'title' => 'Edit Template',
    'description' => 'Edit loader template for loading tab-delimited data',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_modify_template_base_form', 'edit',FALSE),
    'access arguments' => array('administer tripal_bulk_loader'),
    'weight' => -6,
    'type' => MENU_CALLBACK,
  );
  */

  // Add/Edit Record -----
  $items['admin/tripal/loaders/bulk/template/%tblid/add_record'] = array(
    'title' => 'Add Template Field',
    'description' => 'Add a template field to an existing tripal bulk loader template.',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_template_field_form','create_record',5,FALSE),
    'access arguments' => array('administer tripal_bulk_loader'),
    'type' => MENU_CALLBACK,
  );
  $items['admin/tripal/loaders/bulk/template/%tblid/edit_record/%tblid'] = array(
    'title' => 'Edit Template Record',
    'description' => 'Edit a record in an existing tripal bulk loader template.',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_edit_template_record_form',5,7),
    'access arguments' => array('administer tripal_bulk_loader'),
    'type' => MENU_CALLBACK,
  );
  $items['admin/tripal/loaders/bulk/template/%tblid/delete_record/%tblid'] = array(
    'title' => 'Delete Template Record',
    'description' => 'Delete a record in an existing tripal bulk loader template.',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_delete_template_record_form',5,7),
    'access arguments' => array('administer tripal_bulk_loader'),
    'type' => MENU_CALLBACK,
  );
  $items['admin/tripal/loaders/bulk/template/%tblid/duplicate_record/%tblid'] = array(
    'title' => 'Duplicate Template Record',
    'description' => 'Duplicate a record in an existing tripal bulk loader template.',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_duplicate_template_record_form',5,7),
    'access arguments' => array('administer tripal_bulk_loader'),
    'type' => MENU_CALLBACK,
  );

  // Add/Edit Field ------
  $items['admin/tripal/loaders/bulk/template/%tblid/add_field/%tblid'] = array(
    'title' => 'Add Template Field',
    'description' => 'Add a template field to an existing tripal bulk loader template.',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_template_field_form','create',5,7),
    'access arguments' => array('administer tripal_bulk_loader'),
    'type' => MENU_CALLBACK,
  );
  $items['admin/tripal/loaders/bulk/template/%tblid/edit_field/%tblid/%tblid'] = array(
    'title' => 'Edit Template Field',
    'description' => 'Edit an existing field from a tripal bulk loader template.',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_template_field_form','edit',5,7,8),
    'access arguments' => array('administer tripal_bulk_loader'),
    'type' => MENU_CALLBACK,
  );
  $items['admin/tripal/loaders/bulk/template/%tblid/delete_field/%tblid/%tblid'] = array(
    'title' => 'Delete Template Field',
    'description' => 'Delete an existing field from a tripal bulk loader template.',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_delete_template_field_form',5,7,8),
    'access arguments' => array('administer tripal_bulk_loader'),
    'type' => MENU_CALLBACK,
  );

  // Delete Template -----
  $items['admin/tripal/loaders/bulk/template/%tblid/delete'] = array(
    'title' => 'Delete Template',
    'description' => 'Delete bulk loader template',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_delete_template_base_form',5),
    'access arguments' => array('administer tripal_bulk_loader'),
    'weight' => -4,
    'type' => MENU_CALLBACK,
  );
  // Import/Export ---------
  $items['admin/tripal/loaders/bulk/templates/import'] = array(
    'title' => 'Import Template',
    'description' => 'Import Loaders',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_import_template_form'),
    'access arguments' => array('administer tripal_bulk_loader'),
    'weight' => 2,
    'type' => MENU_CALLBACK,
  );
  $items['admin/tripal/loaders/bulk/template/%tblid/export'] = array(
    'title' => 'Export Template',
    'description' => 'Export Loaders',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('tripal_bulk_loader_export_template_form', 5),
    'access arguments' => array('administer tripal_bulk_loader'),
    'weight' => 4,
    'type' => MENU_CALLBACK,
  );

  return $items;
}

function tblid_to_arg($arg, $map, $index) {
  if (preg_match('/^\d+$/', $arg)) {
    return $arg;
  }
}

/**
 * Implements hook_views_api()
 *
 * Purpose: Essentially this hook tells drupal that there is views support for
 *  for this module which then includes tripal_views.views.inc where all the
 *  views integration code is
 *
 * @ingroup tripal_views
 */
function tripal_bulk_loader_views_api() {
  return array(
    'api' => 3.0,
  );
}

/**
 * Implements hook_theme
 *
 * @ingroup tripal_bulk_loader
 */
function tripal_bulk_loader_theme() {
  return array(
    'node__tripal_bulk_loader' => array(
      'template' => 'node--tripal-bulk-loader',
      'path' => drupal_get_path('module', 'tripal_bulk_loader') . '/theme',
      'render element' => 'node',
      'base hook' => 'node',
    ),
    'tripal_bulk_loader_template' => array(
      'variables' => array('template_id' => NULL),
      'template' => 'tripal_bulk_loader_template',
      'path' => drupal_get_path('module', 'tripal_bulk_loader') . '/theme',
    ),
    'tripal_bulk_loader_modify_template_base_form' => array(
      'template' => 'tripal_bulk_loader_modify_template_base_form',
      'path' => drupal_get_path('module', 'tripal_bulk_loader') . '/theme',
      'render element' => 'form'
    ),
    'tripal_bulk_loader_field_regex_fieldset' => array(
      'file' => 'theme/tripal_bulk_loader.theme.inc',
      'function' => 'tripal_bulk_loader_field_regex_fieldset',
      'render element' => 'element'
    ),
    'tripal_bulk_loader_admin' => array(
      'template' => 'tripal_bulk_loader_admin',
      'path' => drupal_get_path('module', 'tripal_bulk_loader') . '/theme',
    ),
  );
}

/**
 * Implements hook_perm
 *
 * @ingroup tripal_bulk_loader
 */
function tripal_bulk_loader_permission() {
  return array(
      'access tripal_bulk_loader' => array(
        'title' => t('View Tripal Bulk Loading Jobs'),
        'description' => t('Permission to view Tripal Bulk Loader Nodes')
      ),
      'create tripal_bulk_loader' => array(
        'title' => t('Create Tripal Bulk Loading Jobs'),
        'description' => t('Permission to create Tripal Bulk Loader Nodes')
      ),
      'edit tripal_bulk_loader' => array(
        'title' => t('Edit Tripal Bulk Loading Jobs'),
        'description' => t('Permission to edit Tripal Bulk Loader Nodes')
      ),
      'delete tripal_bulk_loader' => array(
        'title' => t('Delete Tripal Bulk Loading Jobs'),
        'description' => t('Permission to delete Tripal Bulk Loader Nodes')
      ),
      'administer tripal_bulk_loader' => array(
        'title' => t('Administrate Tripal Bulk Loader'),
        'description' => t('Permission to administrate the Tripal Bulk Loader including template management.')
      ),
  );
}

/**
 * Get the progress of the current constant set from the progress file
 *
 * When transactions are used, database updates to drupal cannot be made. Thus a separate
 * method to keep track of progress was implemented: save a period to the file for each
 * record successfully inserted; each line in the file represents a processed line.
 *
 * @param $job_id
 *   The id of the tripal job to check the progress of
 * @param $node
 *   The tripal_bulk_loader node associated with the job
 *
 * @return
 *   An array with the following keys:
 *     num_lines = the number of lines in the file processed so far
 *     total_lines = the total number of lines in the input file
 *     percent_file = the percent the input file has been loaded
 *     num_records = the number of records successfully inserted
 *
 * @ingroup tripal_bulk_loader
 */
function tripal_bulk_loader_progess_file_get_progress($job_id, $update_progress = TRUE) {
  $filename = '/tmp/tripal_bulk_loader_progress-' . $job_id . '.out';
  if (!file_exists($filename)) {
    return (object) array();
  }

  $num_lines = trim(`wc --lines < $filename`);
  $num_records = trim(`grep -o "." $filename | wc --lines`);

  $job = db_query("SELECT j.*, b.file, b.file_has_header, c.num as num_constant_sets
                              FROM {tripal_jobs} j
                              LEFT JOIN {tripal_bulk_loader} b ON b.job_id=j.job_id
                              LEFT JOIN (
                                  SELECT nid, count(distinct(group_id)) as num
                                  FROM {tripal_bulk_loader_constants}
                                  GROUP BY nid
                                ) c ON c.nid=b.nid
                              WHERE j.job_id=:job", array(':job' =>$job_id))->execute();
  if ($job->num_constant_sets) {
    $num_constant_sets_loaded = round($job->progress / (100 / $job->num_constant_sets), 4);

    // If the next constant set has started loading
    if ($job->num_constant_sets != $num_constant_sets_loaded) {

      // total lines in input file
      $total_lines = trim(`wc --lines < $job->file`);
      if ($job->file_has_header) {
        $total_lines--;
      }

      // percent of the current constant set loaded
      $percent = round($num_lines/$total_lines * 100, 2);

      // percent of the total job = (<# fully loaded constant sets> * 100 )
      //                           + <percent of current constant set>
      //                           / <total number of constant sets>
      $total_percent = (($num_constant_sets_loaded * 100) + $percent) / $job->num_constant_sets;

      // update the progress of the job
      if ($update_progress AND ($percent != 0 OR $percent != 100)) {
        tripal_job_set_progress($job_id, round($total_percent, 0));
      }
    }
  }

  return (object) array(
    'num_lines' => $num_lines,
    'total_lines' => $total_lines,
    'percent_file' => $percent,
    'num_constant_sets_loaded' => $num_constant_sets_loaded,
    'total_percent' => $total_percent,
    'num_records' => $num_records
  );
}

/**
 * Implements hook_job_describe_args()
 * Specifically to make viewing past tripal jobs more readable for jobs registered by this module
 *
 * @params $callback
 *   The callback passed into tripal_add_job()
 * @param $args
 *   The arguements passed into tripal_add_job()
 * @return
 *   An array where keys are the human readable headers describing each arguement
 *   and the value is the aguement passed in after formatting
 *
 * @ingroup tripal_bulk_loader
 */
function tripal_bulk_loader_job_describe_args($callback, $args) {

  $new_args = array();
  if ($callback == 'tripal_bulk_loader_load_data') {
    //1st arg is the nid for a bulk loader node
    $node = node_load($args[0]);
    $new_args['Bulk Loading Job'] = l($node->title, 'node/' . $args[0]);
    return $new_args;
  }

}

/**
 * Implements hook_coder_ignore().
 * Defines the path to the file (tripal_bulk_loader.coder_ignores.txt) where ignore rules for coder are stored
 *
 * @ingroup tripal_bulk_loader
 */
function tripal_bulk_loader_coder_ignore() {
  return array(
    'path' => drupal_get_path('module', 'tripal_bulk_loader'),
    'line prefix' => drupal_get_path('module', 'tripal_bulk_loader'),
  );
}