tripal_bulk_loader.module 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484
  1. <?php
  2. /**
  3. * @file
  4. * Provides general functions for the tripal bulk loader.
  5. *
  6. * @ingroup tripal_bulk_loader
  7. */
  8. /**
  9. * @defgroup tripal_bulk_loader Tripal Bulk Loader Module
  10. * @ingroup tripal_modules
  11. * @{
  12. * Functions implementing the Tripal Generic tab-delimited chado data loader
  13. * @}
  14. */
  15. // Loader
  16. require_once 'includes/tripal_bulk_loader.loader.inc';
  17. // Node
  18. require_once 'includes/tripal_bulk_loader.chado_node.inc';
  19. require_once 'includes/tripal_bulk_loader.constants.inc';
  20. // Administration
  21. require_once 'includes/tripal_bulk_loader.admin.inc';
  22. require_once 'includes/tripal_bulk_loader.admin.templates.inc';
  23. // API
  24. require_once 'api/tripal_bulk_loader.api.templates.inc';
  25. require_once 'api/tripal_bulk_loader.DEPRECATED.inc';
  26. /**
  27. * Implements hook_init().
  28. * Used to add stylesheets and javascript files to the header
  29. *
  30. * @ingroup tripal_bulk_loader
  31. */
  32. function tripal_bulk_loader_init() {
  33. }
  34. /**
  35. * Implements hook_menu().
  36. *
  37. * @ingroup tripal_bulk_loader
  38. */
  39. function tripal_bulk_loader_menu() {
  40. $items = array();
  41. // Bulk Loading Job Node
  42. $items['node/%node/constants/%/edit'] = array(
  43. 'title' => 'Edit Constant Set',
  44. 'description' => 'Edit a group of constants associated with the current bulk loader',
  45. 'page callback' => 'drupal_get_form',
  46. 'page arguments' => array('tripal_bulk_loader_edit_constant_set_form', 1, 3),
  47. 'access arguments' => array('administer tripal_bulk_loader'),
  48. 'type' => MENU_CALLBACK,
  49. );
  50. $items['node/%node/constants/%/delete'] = array(
  51. 'title' => 'Delete Constant Set',
  52. 'description' => 'Delete a group of constants associated with the current bulk loader',
  53. 'page callback' => 'drupal_get_form',
  54. 'page arguments' => array('tripal_bulk_loader_delete_constant_set_form', 1, 3),
  55. 'access arguments' => array('administer tripal_bulk_loader'),
  56. 'type' => MENU_CALLBACK,
  57. );
  58. // Admin pages -----------------
  59. $items['admin/tripal/loaders/bulk'] = array(
  60. 'title' => 'Chado Bulk Loader',
  61. 'description' => 'Templates for loading tab-delimited data',
  62. 'page callback' => 'tripal_bulk_loader_admin_jobs_listing',
  63. 'access arguments' => array('administer tripal_bulk_loader'),
  64. 'type' => MENU_NORMAL_ITEM,
  65. );
  66. $items['admin/tripal/loaders/bulk/configure'] = array(
  67. 'title' => 'Configure',
  68. 'description' => 'Configuration of global options related to bulk loading jobs',
  69. 'page callback' => 'drupal_get_form',
  70. 'page arguments' => array('tripal_bulk_loader_configuration_form'),
  71. 'access arguments' => array('administer tripal_bulk_loader'),
  72. 'weight' => 8,
  73. 'type' => MENU_LOCAL_TASK,
  74. );
  75. $items['admin/tripal/loaders/bulk/help'] = array(
  76. 'title' => 'Help',
  77. 'description' => "A description of the Tripal Bulk Loader module including a short description of it's usage.",
  78. 'page callback' => 'theme',
  79. 'page arguments' => array('tripal_bulk_loader_help'),
  80. 'access arguments' => array('administer tripal_bulk_loader'),
  81. 'weight' => 10,
  82. 'type' => MENU_LOCAL_TASK,
  83. );
  84. // Enable View Callbacks
  85. $items['admin/tripal/loaders/bulk/views/jobs/enable'] = array(
  86. 'title' => 'Enable Jobs Administrative View',
  87. 'description' => 'Enable Jobs Administrative View',
  88. 'page callback' => 'tripal_enable_view',
  89. 'page arguments' => array('tripal_bulk_loading_jobs','admin/tripal/loaders/bulk'),
  90. 'access arguments' => array('administer tripal_bulk_loader'),
  91. 'type' => MENU_CALLBACK,
  92. );
  93. $items['admin/tripal/loaders/bulk/views/templates/enable'] = array(
  94. 'title' => 'Enable Templates Administrative View',
  95. 'description' => 'Enable Templates Administrative View',
  96. 'page callback' => 'tripal_enable_view',
  97. 'page arguments' => array('tripal_bulk_loader_templates', 'admin/tripal/loaders/bulk'),
  98. 'access arguments' => array('administer tripal_bulk_loader'),
  99. 'type' => MENU_CALLBACK,
  100. );
  101. // Create/Edit Template --------
  102. $items['admin/tripal/loaders/bulk/template/create'] = array(
  103. 'title' => 'Create Template',
  104. 'description' => 'Create loader template for loading tab-delimited data',
  105. 'page callback' => 'drupal_get_form',
  106. 'page arguments' => array('tripal_bulk_loader_modify_template_base_form', 'create'),
  107. 'access arguments' => array('administer tripal_bulk_loader'),
  108. 'weight' => -8,
  109. 'type' => MENU_CALLBACK,
  110. );
  111. $items['admin/tripal/loaders/bulk/template/%tblid/edit'] = array(
  112. 'title' => 'Edit Template',
  113. 'description' => 'Edit loader template for loading tab-delimited data',
  114. 'page callback' => 'drupal_get_form',
  115. 'page arguments' => array('tripal_bulk_loader_modify_template_base_form', 'edit', 5),
  116. 'access arguments' => array('administer tripal_bulk_loader'),
  117. 'type' => MENU_CALLBACK,
  118. );
  119. /**
  120. $items['admin/tripal/loaders/bulk/template/edit'] = array(
  121. 'title' => 'Edit Template',
  122. 'description' => 'Edit loader template for loading tab-delimited data',
  123. 'page callback' => 'drupal_get_form',
  124. 'page arguments' => array('tripal_bulk_loader_modify_template_base_form', 'edit',FALSE),
  125. 'access arguments' => array('administer tripal_bulk_loader'),
  126. 'weight' => -6,
  127. 'type' => MENU_CALLBACK,
  128. );
  129. */
  130. // Add/Edit Record -----
  131. $items['admin/tripal/loaders/bulk/template/%tblid/add_record'] = array(
  132. 'title' => 'Add Template Field',
  133. 'description' => 'Add a template field to an existing tripal bulk loader template.',
  134. 'page callback' => 'drupal_get_form',
  135. 'page arguments' => array('tripal_bulk_loader_template_field_form', 'create_record', 5, FALSE),
  136. 'access arguments' => array('administer tripal_bulk_loader'),
  137. 'type' => MENU_CALLBACK,
  138. );
  139. $items['admin/tripal/loaders/bulk/template/%tblid/edit_record/%tblid'] = array(
  140. 'title' => 'Edit Template Record',
  141. 'description' => 'Edit a record in an existing tripal bulk loader template.',
  142. 'page callback' => 'drupal_get_form',
  143. 'page arguments' => array('tripal_bulk_loader_edit_template_record_form', 5, 7),
  144. 'access arguments' => array('administer tripal_bulk_loader'),
  145. 'type' => MENU_CALLBACK,
  146. );
  147. $items['admin/tripal/loaders/bulk/template/%tblid/delete_record/%tblid'] = array(
  148. 'title' => 'Delete Template Record',
  149. 'description' => 'Delete a record in an existing tripal bulk loader template.',
  150. 'page callback' => 'drupal_get_form',
  151. 'page arguments' => array('tripal_bulk_loader_delete_template_record_form', 5, 7),
  152. 'access arguments' => array('administer tripal_bulk_loader'),
  153. 'type' => MENU_CALLBACK,
  154. );
  155. $items['admin/tripal/loaders/bulk/template/%tblid/duplicate_record/%tblid'] = array(
  156. 'title' => 'Duplicate Template Record',
  157. 'description' => 'Duplicate a record in an existing tripal bulk loader template.',
  158. 'page callback' => 'drupal_get_form',
  159. 'page arguments' => array('tripal_bulk_loader_duplicate_template_record_form', 5, 7),
  160. 'access arguments' => array('administer tripal_bulk_loader'),
  161. 'type' => MENU_CALLBACK,
  162. );
  163. // Add/Edit Field ------
  164. $items['admin/tripal/loaders/bulk/template/%tblid/add_field/%tblid'] = array(
  165. 'title' => 'Add Template Field',
  166. 'description' => 'Add a template field to an existing tripal bulk loader template.',
  167. 'page callback' => 'drupal_get_form',
  168. 'page arguments' => array('tripal_bulk_loader_template_field_form','create', 5, 7),
  169. 'access arguments' => array('administer tripal_bulk_loader'),
  170. 'type' => MENU_CALLBACK,
  171. );
  172. $items['admin/tripal/loaders/bulk/template/%tblid/edit_field/%tblid/%tblid'] = array(
  173. 'title' => 'Edit Template Field',
  174. 'description' => 'Edit an existing field from a tripal bulk loader template.',
  175. 'page callback' => 'drupal_get_form',
  176. 'page arguments' => array('tripal_bulk_loader_template_field_form','edit', 5, 7, 8),
  177. 'access arguments' => array('administer tripal_bulk_loader'),
  178. 'type' => MENU_CALLBACK,
  179. );
  180. $items['admin/tripal/loaders/bulk/template/%tblid/delete_field/%tblid/%tblid'] = array(
  181. 'title' => 'Delete Template Field',
  182. 'description' => 'Delete an existing field from a tripal bulk loader template.',
  183. 'page callback' => 'drupal_get_form',
  184. 'page arguments' => array('tripal_bulk_loader_delete_template_field_form', 5, 7, 8),
  185. 'access arguments' => array('administer tripal_bulk_loader'),
  186. 'type' => MENU_CALLBACK,
  187. );
  188. // Delete Template -----
  189. $items['admin/tripal/loaders/bulk/template/%tblid/delete'] = array(
  190. 'title' => 'Delete Template',
  191. 'description' => 'Delete bulk loader template',
  192. 'page callback' => 'drupal_get_form',
  193. 'page arguments' => array('tripal_bulk_loader_delete_template_base_form', 5),
  194. 'access arguments' => array('administer tripal_bulk_loader'),
  195. 'weight' => -4,
  196. 'type' => MENU_CALLBACK,
  197. );
  198. // Import/Export ---------
  199. $items['admin/tripal/loaders/bulk/templates/import'] = array(
  200. 'title' => 'Import Template',
  201. 'description' => 'Import Loaders',
  202. 'page callback' => 'drupal_get_form',
  203. 'page arguments' => array('tripal_bulk_loader_import_template_form'),
  204. 'access arguments' => array('administer tripal_bulk_loader'),
  205. 'weight' => 2,
  206. 'type' => MENU_CALLBACK,
  207. );
  208. $items['admin/tripal/loaders/bulk/template/%tblid/export'] = array(
  209. 'title' => 'Export Template',
  210. 'description' => 'Export Loaders',
  211. 'page callback' => 'drupal_get_form',
  212. 'page arguments' => array('tripal_bulk_loader_export_template_form', 5),
  213. 'access arguments' => array('administer tripal_bulk_loader'),
  214. 'weight' => 4,
  215. 'type' => MENU_CALLBACK,
  216. );
  217. return $items;
  218. }
  219. /**
  220. * Implements hook_to_arg().
  221. * Ensures the arguement for the bulk loader templates path is correct
  222. *
  223. * @ingroup tripal_bulk_loader
  224. */
  225. function tblid_to_arg($arg, $map, $index) {
  226. if (preg_match('/^(\d+|O)$/', $arg)) {
  227. return $arg;
  228. }
  229. }
  230. /**
  231. * Implements hook_load() for the menu system.
  232. *
  233. * Ensures that a number is passed to the form. We use the letter o in our path instead
  234. * of the number 0 because the drupal menu system has a bug that doesn't allow 0 as
  235. * the only character in the path.
  236. *
  237. * @ingroup tripal_bulk_loader
  238. */
  239. function tblid_load($tblid_id) {
  240. if (preg_match('/O/',$tblid_id)) {
  241. // This ensures that the number 0 is sent to the form as the correct arg
  242. return 0;
  243. }
  244. else {
  245. return $tblid_id;
  246. }
  247. }
  248. /**
  249. * Implements hook_views_api().
  250. *
  251. * Essentially this hook tells drupal that there is views support for
  252. * for this module which then includes tripal_views.views.inc where all the
  253. * views integration code is.
  254. *
  255. * @ingroup tripal_bulk_loader
  256. */
  257. function tripal_bulk_loader_views_api() {
  258. return array(
  259. 'api' => 3.0,
  260. );
  261. }
  262. /**
  263. * Implements hook_theme().
  264. *
  265. * @ingroup tripal_bulk_loader
  266. */
  267. function tripal_bulk_loader_theme($existing, $type, $theme, $path) {
  268. $core_path = drupal_get_path('module', 'tripal_core');
  269. return array(
  270. 'node__tripal_bulk_loader' => array(
  271. 'template' => 'node--chado-generic',
  272. 'render element' => 'node',
  273. 'base hook' => 'node',
  274. 'path' => "$core_path/theme/templates",
  275. ),
  276. 'tripal_bulk_loader_base' => array(
  277. 'variables' => array('node' => NULL),
  278. 'template' => 'tripal_bulk_loader_base',
  279. 'path' => "$path/theme/templates",
  280. ),
  281. 'tripal_bulk_loader_fields' => array(
  282. 'variables' => array('node' => NULL),
  283. 'template' => 'tripal_bulk_loader_fields',
  284. 'path' => "$path/theme/templates",
  285. ),
  286. // form element themes
  287. 'tripal_bulk_loader_modify_template_base_form_records' => array(
  288. 'template' => 'tripal_bulk_loader_modify_template_base_form_records',
  289. 'path' => "$path/theme/templates",
  290. 'render element' => 'element'
  291. ),
  292. 'tripal_bulk_loader_modify_template_base_form_fields' => array(
  293. 'template' => 'tripal_bulk_loader_modify_template_base_form_fields',
  294. 'path' => "$path/theme/templates",
  295. 'render element' => 'element'
  296. ),
  297. 'tripal_bulk_loader_field_regex_fieldset' => array(
  298. 'file' => 'theme/tripal_bulk_loader.theme.inc',
  299. 'function' => 'tripal_bulk_loader_field_regex_fieldset',
  300. 'render element' => 'element'
  301. ),
  302. 'tripal_bulk_loader_constant_set' => array(
  303. 'function' => 'theme_tripal_bulk_loader_constant_set',
  304. 'variables' => array(
  305. 'nid' => NULL,
  306. 'constants' => NULL,
  307. 'template' => NULL,
  308. 'options' => array()
  309. ),
  310. ),
  311. // help page
  312. 'tripal_bulk_loader_help' => array(
  313. 'template' => 'tripal_bulk_loader_help',
  314. 'path' => "$path/theme/templates",
  315. ),
  316. // themed teaser
  317. 'tripal_bulk_loader_teaser' => array(
  318. 'variables' => array('node' => NULL),
  319. 'template' => 'tripal_bulk_loader_teaser',
  320. 'path' => "$path/theme/templates",
  321. ),
  322. );
  323. }
  324. /**
  325. * Implements hook_permission().
  326. *
  327. * @ingroup tripal_bulk_loader
  328. */
  329. function tripal_bulk_loader_permission() {
  330. return array(
  331. 'access tripal_bulk_loader' => array(
  332. 'title' => t('View Tripal Bulk Loading Jobs'),
  333. 'description' => t('Permission to view Tripal Bulk Loader Nodes')
  334. ),
  335. 'create tripal_bulk_loader' => array(
  336. 'title' => t('Create Tripal Bulk Loading Jobs'),
  337. 'description' => t('Permission to create Tripal Bulk Loader Nodes')
  338. ),
  339. 'edit tripal_bulk_loader' => array(
  340. 'title' => t('Edit Tripal Bulk Loading Jobs'),
  341. 'description' => t('Permission to edit Tripal Bulk Loader Nodes')
  342. ),
  343. 'delete tripal_bulk_loader' => array(
  344. 'title' => t('Delete Tripal Bulk Loading Jobs'),
  345. 'description' => t('Permission to delete Tripal Bulk Loader Nodes')
  346. ),
  347. 'administer tripal_bulk_loader' => array(
  348. 'title' => t('Administrate Tripal Bulk Loader'),
  349. 'description' => t('Permission to administrate the Tripal Bulk Loader including template management.')
  350. ),
  351. );
  352. }
  353. /**
  354. * Get the progress of the current constant set from the progress file
  355. *
  356. * When transactions are used, database updates to drupal cannot be made. Thus a separate
  357. * method to keep track of progress was implemented: save a period to the file for each
  358. * record successfully inserted; each line in the file represents a processed line.
  359. *
  360. * @param $job_id
  361. * The id of the tripal job to check the progress of
  362. * @param $node
  363. * The tripal_bulk_loader node associated with the job
  364. *
  365. * @return
  366. * An array with the following keys:
  367. * num_lines = the number of lines in the file processed so far
  368. * total_lines = the total number of lines in the input file
  369. * percent_file = the percent the input file has been loaded
  370. * num_records = the number of records successfully inserted
  371. *
  372. * @ingroup tripal_bulk_loader
  373. */
  374. function tripal_bulk_loader_progess_file_get_progress($job_id, $update_progress = TRUE) {
  375. $filename = '/tmp/tripal_bulk_loader_progress-' . $job_id . '.out';
  376. if (!file_exists($filename)) {
  377. return (object) array();
  378. }
  379. $num_lines = trim(shell_exec('wc --lines < ' . escapeshellarg($filename)));
  380. $num_records = trim(shell_exec('grep -o "." ' . escapeshellarg($filename) . ' | wc --lines'));
  381. $job = db_query("SELECT j.*, b.file, b.file_has_header, c.num as num_constant_sets
  382. FROM {tripal_jobs} j
  383. LEFT JOIN {tripal_bulk_loader} b ON b.job_id=j.job_id
  384. LEFT JOIN (
  385. SELECT nid, count(distinct(group_id)) as num
  386. FROM {tripal_bulk_loader_constants}
  387. GROUP BY nid
  388. ) c ON c.nid=b.nid
  389. WHERE j.job_id=:job", array(':job' =>$job_id))->execute();
  390. if ($job->num_constant_sets) {
  391. $num_constant_sets_loaded = round($job->progress / (100 / $job->num_constant_sets), 4);
  392. // If the next constant set has started loading
  393. if ($job->num_constant_sets != $num_constant_sets_loaded) {
  394. // total lines in input file
  395. $total_lines = trim(shell_exec('wc --lines < ' . escapeshellarg($job->file)));
  396. if ($job->file_has_header) {
  397. $total_lines--;
  398. }
  399. // percent of the current constant set loaded
  400. $percent = round($num_lines/$total_lines * 100, 2);
  401. // percent of the total job = (<# fully loaded constant sets> * 100 )
  402. // + <percent of current constant set>
  403. // / <total number of constant sets>
  404. $total_percent = (($num_constant_sets_loaded * 100) + $percent) / $job->num_constant_sets;
  405. // update the progress of the job
  406. if ($update_progress AND ($percent != 0 OR $percent != 100)) {
  407. tripal_set_job_progress($job_id, round($total_percent, 0));
  408. }
  409. }
  410. }
  411. return (object) array(
  412. 'num_lines' => $num_lines,
  413. 'total_lines' => $total_lines,
  414. 'percent_file' => $percent,
  415. 'num_constant_sets_loaded' => $num_constant_sets_loaded,
  416. 'total_percent' => $total_percent,
  417. 'num_records' => $num_records
  418. );
  419. }
  420. /**
  421. * Implements hook_job_describe_args()
  422. *
  423. * @ingroup tripal_bulk_loader
  424. */
  425. function tripal_bulk_loader_job_describe_args($callback, $args) {
  426. $new_args = array();
  427. if ($callback == 'tripal_bulk_loader_load_data') {
  428. //1st arg is the nid for a bulk loader node
  429. $node = node_load($args[0]);
  430. $new_args['Bulk Loading Job'] = l($node->title, 'node/' . $args[0]);
  431. return $new_args;
  432. }
  433. }
  434. /**
  435. * Implements hook_coder_ignore().
  436. * Defines the path to the file (tripal_bulk_loader.coder_ignores.txt) where ignore rules for coder are stored
  437. *
  438. * @ingroup tripal_bulk_loader
  439. */
  440. function tripal_bulk_loader_coder_ignore() {
  441. return array(
  442. 'path' => drupal_get_path('module', 'tripal_bulk_loader'),
  443. 'line prefix' => drupal_get_path('module', 'tripal_bulk_loader'),
  444. );
  445. }