<?php
/**
* @file
* Provides general functions for the tripal bulk loader.
*
* @ingroup tripal_bulk_loader
*/
/**
* @defgroup tripal_bulk_loader Tripal Bulk Loader Module
* @ingroup tripal_modules
* @{
* Functions implementing the Tripal Generic tab-delimited chado data loader
* @}
*/
// Loader
require_once 'includes/tripal_bulk_loader.loader.inc';
// Node
require_once 'includes/tripal_bulk_loader.chado_node.inc';
require_once 'includes/tripal_bulk_loader.constants.inc';
// Administration
require_once 'includes/tripal_bulk_loader.admin.inc';
require_once 'includes/tripal_bulk_loader.admin.templates.inc';
// API
require_once 'api/tripal_bulk_loader.api.templates.inc';
require_once 'api/tripal_bulk_loader.DEPRECATED.inc';
/**
* Implements hook_init().
* Used to add stylesheets and javascript files to the header
*
* @ingroup tripal_bulk_loader
*/
function tripal_bulk_loader_init() {
}
/**
* Implements hook_menu().
*
* @ingroup tripal_bulk_loader
*/
function tripal_bulk_loader_menu() {
$items = array();
// Bulk Loading Job Node
$items['node/%node/constants/%/edit'] = array(
'title' => 'Edit Constant Set',
'description' => 'Edit a group of constants associated with the current bulk loader',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_edit_constant_set_form', 1, 3),
'access arguments' => array('administer tripal_bulk_loader'),
'type' => MENU_CALLBACK,
);
$items['node/%node/constants/%/delete'] = array(
'title' => 'Delete Constant Set',
'description' => 'Delete a group of constants associated with the current bulk loader',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_delete_constant_set_form', 1, 3),
'access arguments' => array('administer tripal_bulk_loader'),
'type' => MENU_CALLBACK,
);
// Admin pages -----------------
$items['admin/tripal/loaders/bulk'] = array(
'title' => 'Bulk Loader',
'description' => 'Templates for loading tab-delimited data',
'page callback' => 'tripal_bulk_loader_admin_jobs_listing',
'access arguments' => array('administer tripal_bulk_loader'),
'type' => MENU_NORMAL_ITEM,
);
$items['admin/tripal/loaders/bulk/configure'] = array(
'title' => 'Configure',
'description' => 'Configuration of global options related to bulk loading jobs',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_configuration_form'),
'access arguments' => array('administer tripal_bulk_loader'),
'weight' => 8,
'type' => MENU_LOCAL_TASK,
);
$items['admin/tripal/loaders/bulk/help'] = array(
'title' => 'Help',
'description' => "A description of the Tripal Bulk Loader module including a short description of it's usage.",
'page callback' => 'theme',
'page arguments' => array('tripal_bulk_loader_help'),
'access arguments' => array('administer tripal_bulk_loader'),
'weight' => 10,
'type' => MENU_LOCAL_TASK,
);
// Enable View Callbacks
$items['admin/tripal/loaders/bulk/views/jobs/enable'] = array(
'title' => 'Enable Jobs Administrative View',
'description' => 'Enable Jobs Administrative View',
'page callback' => 'tripal_enable_view',
'page arguments' => array('tripal_bulk_loading_jobs','admin/tripal/loaders/bulk'),
'access arguments' => array('administer tripal_bulk_loader'),
'type' => MENU_CALLBACK,
);
$items['admin/tripal/loaders/bulk/views/templates/enable'] = array(
'title' => 'Enable Templates Administrative View',
'description' => 'Enable Templates Administrative View',
'page callback' => 'tripal_enable_view',
'page arguments' => array('tripal_bulk_loader_templates', 'admin/tripal/loaders/bulk'),
'access arguments' => array('administer tripal_bulk_loader'),
'type' => MENU_CALLBACK,
);
// Create/Edit Template --------
$items['admin/tripal/loaders/bulk/template/create'] = array(
'title' => 'Create Template',
'description' => 'Create loader template for loading tab-delimited data',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_modify_template_base_form', 'create'),
'access arguments' => array('administer tripal_bulk_loader'),
'weight' => -8,
'type' => MENU_CALLBACK,
);
$items['admin/tripal/loaders/bulk/template/%tblid/edit'] = array(
'title' => 'Edit Template',
'description' => 'Edit loader template for loading tab-delimited data',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_modify_template_base_form', 'edit', 5),
'access arguments' => array('administer tripal_bulk_loader'),
'type' => MENU_CALLBACK,
);
/**
$items['admin/tripal/loaders/bulk/template/edit'] = array(
'title' => 'Edit Template',
'description' => 'Edit loader template for loading tab-delimited data',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_modify_template_base_form', 'edit',FALSE),
'access arguments' => array('administer tripal_bulk_loader'),
'weight' => -6,
'type' => MENU_CALLBACK,
);
*/
// Add/Edit Record -----
$items['admin/tripal/loaders/bulk/template/%tblid/add_record'] = array(
'title' => 'Add Template Field',
'description' => 'Add a template field to an existing tripal bulk loader template.',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_template_field_form', 'create_record', 5, FALSE),
'access arguments' => array('administer tripal_bulk_loader'),
'type' => MENU_CALLBACK,
);
$items['admin/tripal/loaders/bulk/template/%tblid/edit_record/%tblid'] = array(
'title' => 'Edit Template Record',
'description' => 'Edit a record in an existing tripal bulk loader template.',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_edit_template_record_form', 5, 7),
'access arguments' => array('administer tripal_bulk_loader'),
'type' => MENU_CALLBACK,
);
$items['admin/tripal/loaders/bulk/template/%tblid/delete_record/%tblid'] = array(
'title' => 'Delete Template Record',
'description' => 'Delete a record in an existing tripal bulk loader template.',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_delete_template_record_form', 5, 7),
'access arguments' => array('administer tripal_bulk_loader'),
'type' => MENU_CALLBACK,
);
$items['admin/tripal/loaders/bulk/template/%tblid/duplicate_record/%tblid'] = array(
'title' => 'Duplicate Template Record',
'description' => 'Duplicate a record in an existing tripal bulk loader template.',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_duplicate_template_record_form', 5, 7),
'access arguments' => array('administer tripal_bulk_loader'),
'type' => MENU_CALLBACK,
);
// Add/Edit Field ------
$items['admin/tripal/loaders/bulk/template/%tblid/add_field/%tblid'] = array(
'title' => 'Add Template Field',
'description' => 'Add a template field to an existing tripal bulk loader template.',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_template_field_form','create', 5, 7),
'access arguments' => array('administer tripal_bulk_loader'),
'type' => MENU_CALLBACK,
);
$items['admin/tripal/loaders/bulk/template/%tblid/edit_field/%tblid/%tblid'] = array(
'title' => 'Edit Template Field',
'description' => 'Edit an existing field from a tripal bulk loader template.',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_template_field_form','edit', 5, 7, 8),
'access arguments' => array('administer tripal_bulk_loader'),
'type' => MENU_CALLBACK,
);
$items['admin/tripal/loaders/bulk/template/%tblid/delete_field/%tblid/%tblid'] = array(
'title' => 'Delete Template Field',
'description' => 'Delete an existing field from a tripal bulk loader template.',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_delete_template_field_form', 5, 7, 8),
'access arguments' => array('administer tripal_bulk_loader'),
'type' => MENU_CALLBACK,
);
// Delete Template -----
$items['admin/tripal/loaders/bulk/template/%tblid/delete'] = array(
'title' => 'Delete Template',
'description' => 'Delete bulk loader template',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_delete_template_base_form', 5),
'access arguments' => array('administer tripal_bulk_loader'),
'weight' => -4,
'type' => MENU_CALLBACK,
);
// Import/Export ---------
$items['admin/tripal/loaders/bulk/templates/import'] = array(
'title' => 'Import Template',
'description' => 'Import Loaders',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_import_template_form'),
'access arguments' => array('administer tripal_bulk_loader'),
'weight' => 2,
'type' => MENU_CALLBACK,
);
$items['admin/tripal/loaders/bulk/template/%tblid/export'] = array(
'title' => 'Export Template',
'description' => 'Export Loaders',
'page callback' => 'drupal_get_form',
'page arguments' => array('tripal_bulk_loader_export_template_form', 5),
'access arguments' => array('administer tripal_bulk_loader'),
'weight' => 4,
'type' => MENU_CALLBACK,
);
return $items;
}
/**
* Implements hook_to_arg().
* Ensures the arguement for the bulk loader templates path is correct
*
* @ingroup tripal_bulk_loader
*/
function tblid_to_arg($arg, $map, $index) {
if (preg_match('/^(\d+|O)$/', $arg)) {
return $arg;
}
}
/**
* Implements hook_load() for the menu system.
*
* Ensures that a number is passed to the form. We use the letter o in our path instead
* of the number 0 because the drupal menu system has a bug that doesn't allow 0 as
* the only character in the path.
*
* @ingroup tripal_bulk_loader
*/
function tblid_load($tblid_id) {
if (preg_match('/O/',$tblid_id)) {
// This ensures that the number 0 is sent to the form as the correct arg
return 0;
}
else {
return $tblid_id;
}
}
/**
* Implements hook_views_api().
*
* Essentially this hook tells drupal that there is views support for
* for this module which then includes tripal_views.views.inc where all the
* views integration code is.
*
* @ingroup tripal_bulk_loader
*/
function tripal_bulk_loader_views_api() {
return array(
'api' => 3.0,
);
}
/**
* Implements hook_theme().
*
* @ingroup tripal_bulk_loader
*/
function tripal_bulk_loader_theme($existing, $type, $theme, $path) {
$core_path = drupal_get_path('module', 'tripal_core');
return array(
'node__tripal_bulk_loader' => array(
'template' => 'node--chado-generic',
'render element' => 'node',
'base hook' => 'node',
'path' => "$core_path/theme/templates",
),
'tripal_bulk_loader_base' => array(
'variables' => array('node' => NULL),
'template' => 'tripal_bulk_loader_base',
'path' => "$path/theme/templates",
),
'tripal_bulk_loader_fields' => array(
'variables' => array('node' => NULL),
'template' => 'tripal_bulk_loader_fields',
'path' => "$path/theme/templates",
),
// form element themes
'tripal_bulk_loader_modify_template_base_form' => array(
'template' => 'tripal_bulk_loader_modify_template_base_form',
'path' => "$path/theme/templates",
'render element' => 'form'
),
'tripal_bulk_loader_field_regex_fieldset' => array(
'file' => 'theme/tripal_bulk_loader.theme.inc',
'function' => 'tripal_bulk_loader_field_regex_fieldset',
'render element' => 'element'
),
'tripal_bulk_loader_constant_set' => array(
'function' => 'theme_tripal_bulk_loader_constant_set',
'variables' => array(
'nid' => NULL,
'constants' => NULL,
'template' => NULL,
'options' => array()
),
),
// help page
'tripal_bulk_loader_help' => array(
'template' => 'tripal_bulk_loader_help',
'path' => "$path/theme/templates",
),
// themed teaser
'tripal_bulk_loader_teaser' => array(
'variables' => array('node' => NULL),
'template' => 'tripal_bulk_loader_teaser',
'path' => "$path/theme/templates",
),
);
}
/**
* Implements hook_permission().
*
* @ingroup tripal_bulk_loader
*/
function tripal_bulk_loader_permission() {
return array(
'access tripal_bulk_loader' => array(
'title' => t('View Tripal Bulk Loading Jobs'),
'description' => t('Permission to view Tripal Bulk Loader Nodes')
),
'create tripal_bulk_loader' => array(
'title' => t('Create Tripal Bulk Loading Jobs'),
'description' => t('Permission to create Tripal Bulk Loader Nodes')
),
'edit tripal_bulk_loader' => array(
'title' => t('Edit Tripal Bulk Loading Jobs'),
'description' => t('Permission to edit Tripal Bulk Loader Nodes')
),
'delete tripal_bulk_loader' => array(
'title' => t('Delete Tripal Bulk Loading Jobs'),
'description' => t('Permission to delete Tripal Bulk Loader Nodes')
),
'administer tripal_bulk_loader' => array(
'title' => t('Administrate Tripal Bulk Loader'),
'description' => t('Permission to administrate the Tripal Bulk Loader including template management.')
),
);
}
/**
* Get the progress of the current constant set from the progress file
*
* When transactions are used, database updates to drupal cannot be made. Thus a separate
* method to keep track of progress was implemented: save a period to the file for each
* record successfully inserted; each line in the file represents a processed line.
*
* @param $job_id
* The id of the tripal job to check the progress of
* @param $node
* The tripal_bulk_loader node associated with the job
*
* @return
* An array with the following keys:
* num_lines = the number of lines in the file processed so far
* total_lines = the total number of lines in the input file
* percent_file = the percent the input file has been loaded
* num_records = the number of records successfully inserted
*
* @ingroup tripal_bulk_loader
*/
function tripal_bulk_loader_progess_file_get_progress($job_id, $update_progress = TRUE) {
$filename = '/tmp/tripal_bulk_loader_progress-' . $job_id . '.out';
if (!file_exists($filename)) {
return (object) array();
}
$num_lines = trim(shell_exec('wc --lines < ' . escapeshellarg($filename)));
$num_records = trim(shell_exec('grep -o "." ' . escapeshellarg($filename) . ' | wc --lines'));
$job = db_query("SELECT j.*, b.file, b.file_has_header, c.num as num_constant_sets
FROM {tripal_jobs} j
LEFT JOIN {tripal_bulk_loader} b ON b.job_id=j.job_id
LEFT JOIN (
SELECT nid, count(distinct(group_id)) as num
FROM {tripal_bulk_loader_constants}
GROUP BY nid
) c ON c.nid=b.nid
WHERE j.job_id=:job", array(':job' =>$job_id))->execute();
if ($job->num_constant_sets) {
$num_constant_sets_loaded = round($job->progress / (100 / $job->num_constant_sets), 4);
// If the next constant set has started loading
if ($job->num_constant_sets != $num_constant_sets_loaded) {
// total lines in input file
$total_lines = trim(shell_exec('wc --lines < ' . escapeshellarg($job->file)));
if ($job->file_has_header) {
$total_lines--;
}
// percent of the current constant set loaded
$percent = round($num_lines/$total_lines * 100, 2);
// percent of the total job = (<# fully loaded constant sets> * 100 )
// + <percent of current constant set>
// / <total number of constant sets>
$total_percent = (($num_constant_sets_loaded * 100) + $percent) / $job->num_constant_sets;
// update the progress of the job
if ($update_progress AND ($percent != 0 OR $percent != 100)) {
tripal_set_job_progress($job_id, round($total_percent, 0));
}
}
}
return (object) array(
'num_lines' => $num_lines,
'total_lines' => $total_lines,
'percent_file' => $percent,
'num_constant_sets_loaded' => $num_constant_sets_loaded,
'total_percent' => $total_percent,
'num_records' => $num_records
);
}
/**
* Implements hook_job_describe_args()
* Specifically to make viewing past tripal jobs more readable for jobs registered by this module
*
* @params $callback
* The callback passed into tripal_add_job()
* @param $args
* The arguements passed into tripal_add_job()
* @return
* An array where keys are the human readable headers describing each arguement
* and the value is the aguement passed in after formatting
*
* @ingroup tripal_bulk_loader
*/
function tripal_bulk_loader_job_describe_args($callback, $args) {
$new_args = array();
if ($callback == 'tripal_bulk_loader_load_data') {
//1st arg is the nid for a bulk loader node
$node = node_load($args[0]);
$new_args['Bulk Loading Job'] = l($node->title, 'node/' . $args[0]);
return $new_args;
}
}
/**
* Implements hook_coder_ignore().
* Defines the path to the file (tripal_bulk_loader.coder_ignores.txt) where ignore rules for coder are stored
*
* @ingroup tripal_bulk_loader
*/
function tripal_bulk_loader_coder_ignore() {
return array(
'path' => drupal_get_path('module', 'tripal_bulk_loader'),
'line prefix' => drupal_get_path('module', 'tripal_bulk_loader'),
);
}