TaxonomyImporter.inc 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943
  1. <?php
  2. class TaxonomyImporter extends TripalImporter {
  3. /**
  4. * The name of this loader. This name will be presented to the site
  5. * user.
  6. */
  7. public static $name = 'Chado NCBI Taxonomy Loader';
  8. /**
  9. * The machine name for this loader. This name will be used to construct
  10. * the URL for the loader.
  11. */
  12. public static $machine_name = 'chado_taxonomy';
  13. /**
  14. * A brief description for this loader. This description will be
  15. * presented to the site user.
  16. */
  17. public static $description = 'Imports new organisms from NCBI using taxonomy IDs, or loads taxonomic details about existing organisms.';
  18. /**
  19. * An array containing the extensions of allowed file types.
  20. */
  21. public static $file_types = [];
  22. /**
  23. * Provides information to the user about the file upload. Typically this
  24. * may include a description of the file types allowed.
  25. */
  26. public static $upload_description = '';
  27. /**
  28. * The title that should appear above the upload button.
  29. */
  30. public static $upload_title = 'File Upload';
  31. /**
  32. * If the loader should require an analysis record. To maintain provenance
  33. * we should always indiate where the data we are uploading comes from.
  34. * The method that Tripal attempts to use for this by associating upload files
  35. * with an analysis record. The analysis record provides the details for
  36. * how the file was created or obtained. Set this to FALSE if the loader
  37. * should not require an analysis when loading. if $use_analysis is set to
  38. * true then the form values will have an 'analysis_id' key in the $form_state
  39. * array on submitted forms.
  40. */
  41. public static $use_analysis = FALSE;
  42. /**
  43. * If the $use_analysis value is set above then this value indicates if the
  44. * analysis should be required.
  45. */
  46. public static $require_analysis = FALSE;
  47. /**
  48. * Text that should appear on the button at the bottom of the importer
  49. * form.
  50. */
  51. public static $button_text = 'Import from NCBI Taxonomy';
  52. /**
  53. * Indicates the methods that the file uploader will support.
  54. */
  55. public static $methods = [
  56. // Allow the user to upload a file to the server.
  57. 'file_upload' => FALSE,
  58. // Allow the user to provide the path on the Tripal server for the file.
  59. 'file_local' => FALSE,
  60. // Allow the user to provide a remote URL for the file.
  61. 'file_remote' => FALSE,
  62. ];
  63. /**
  64. * Indicates if the file must be provided. An example when it may not be
  65. * necessary to require that the user provide a file for uploading if the
  66. * loader keeps track of previous files and makes those available for
  67. * selection.
  68. */
  69. public static $file_required = FALSE;
  70. /**
  71. * The array of arguments used for this loader. Each argument should
  72. * be a separate array containing a machine_name, name, and description
  73. * keys. This information is used to build the help text for the loader.
  74. */
  75. public static $argument_list = [];
  76. /**
  77. * Indicates how many files are allowed to be uploaded. By default this is
  78. * set to allow only one file. Change to any positive number. A value of
  79. * zero indicates an unlimited number of uploaded files are allowed.
  80. */
  81. public static $cardinality = 0;
  82. /**
  83. * Holds the list of all orgainsms currently in Chado. This list
  84. * is needed when checking to see if an organism has already been
  85. * loaded.
  86. */
  87. private $all_orgs = [];
  88. /**
  89. * The record from the Chado phylotree table that refers to this
  90. * Taxonomic tree.
  91. */
  92. private $phylotree = NULL;
  93. /**
  94. * The temporary tree array used by the Tripal Phylotree API for
  95. * importing a new tree.
  96. */
  97. private $tree = NULL;
  98. /**
  99. * @see TripalImporter::form()
  100. */
  101. public function form($form, &$form_state) {
  102. $form['instructions'] = [
  103. '#type' => 'fieldset',
  104. '#title' => 'instructions',
  105. '#description' => t('This form is used to import species from the NCBI
  106. Taxonomy database into this site. Alternatively, it can import details
  107. about organisms from the NCBI Taxonomy database for organisms that
  108. already exist on this site. This loader will also construct
  109. the taxonomic tree for the species loaded.'),
  110. ];
  111. $form['taxonomy_ids'] = [
  112. '#type' => 'textarea',
  113. '#title' => 'Taxonomy ID',
  114. '#description' => t('Please provide a list of NCBI taxonomy IDs separated
  115. by spaces, tabs or new lines.
  116. The information about these organisms will be downloaded and new organism
  117. records will be added to this site.'),
  118. ];
  119. $form['import_existing'] = [
  120. '#type' => 'checkbox',
  121. '#title' => 'Import details for existing species.',
  122. '#description' => t('The NCBI Taxonomic Importer examines the organisms
  123. currently present in the database and queries NCBI for the
  124. taxonomic details. If the importer is able to match the
  125. genus and species with NCBI the species details will be imported,
  126. and a page containing the taxonomic tree will be created.'),
  127. '#default value' => 1,
  128. ];
  129. return $form;
  130. }
  131. /**
  132. * @see TripalImporter::formValidate()
  133. */
  134. public function formValidate($form, &$form_state) {
  135. global $user;
  136. $import_existing = $form_state['values']['import_existing'];
  137. $taxonomy_ids = $form_state['values']['taxonomy_ids'];
  138. // make sure that we have numeric values, one per line.
  139. if ($taxonomy_ids) {
  140. $tax_ids = preg_split("/[\s\n\t\r]+/", $taxonomy_ids);
  141. $bad_ids = [];
  142. foreach ($tax_ids as $tax_id) {
  143. $tax_id = trim($tax_id);
  144. if (!preg_match('/^\d+$/', $tax_id)) {
  145. $bad_ids[] = $tax_id;
  146. }
  147. }
  148. if (count($bad_ids) > 0) {
  149. form_set_error('taxonomy_ids',
  150. t('Taxonomy IDs must be numeric. The following are not valid: "@ids".',
  151. ['@ids' => implode('", "', $bad_ids)]));
  152. }
  153. }
  154. }
  155. /**
  156. * Performs the import.
  157. */
  158. public function run() {
  159. global $site_name;
  160. $arguments = $this->arguments['run_args'];
  161. $taxonomy_ids = $arguments['taxonomy_ids'];
  162. $import_existing = $arguments['import_existing'];
  163. // Get the list of all organisms as we'll need this to lookup existing
  164. // organisms.
  165. if (chado_get_version() > 1.2) {
  166. $sql = "
  167. SELECT O.*, CVT.name as type
  168. FROM {organism} O
  169. LEFT JOIN {cvterm} CVT ON CVT.cvterm_id = O.type_id
  170. ORDER BY O.genus, O.species
  171. ";
  172. }
  173. else {
  174. $sql = "
  175. SELECT O.*, '' as type
  176. FROM {organism} O
  177. ORDER BY O.genus, O.species
  178. ";
  179. }
  180. $results = chado_query($sql);
  181. while ($item = $results->fetchObject()) {
  182. $this->all_orgs[] = $item;
  183. }
  184. // Get the phylotree object.
  185. $this->logMessage('Initializing Tree...');
  186. $this->phylotree = $this->initTree();
  187. $this->logMessage('Rebuilding Tree...');
  188. $this->tree = $this->rebuildTree();
  189. // Clean out the phnylondes for this tree in the event this is a reload
  190. chado_delete_record('phylonode', ['phylotree_id' => $this->phylotree->phylotree_id]);
  191. // Get the taxonomy IDs provided by the user (if any).
  192. $tax_ids = [];
  193. if ($taxonomy_ids) {
  194. $tax_ids = preg_split("/[\s\n\t\r]+/", $taxonomy_ids);
  195. }
  196. // Set the number of items to handle.
  197. if ($taxonomy_ids and $import_existing) {
  198. $this->setTotalItems(count($this->all_orgs) + count($tax_ids));
  199. }
  200. if ($taxonomy_ids and !$import_existing) {
  201. $this->setTotalItems(count($tax_ids));
  202. }
  203. if (!$taxonomy_ids and $import_existing) {
  204. $this->setTotalItems(count($this->all_orgs));
  205. }
  206. $this->setItemsHandled(0);
  207. // If the user wants to import new taxonomy IDs then do that.
  208. if ($taxonomy_ids) {
  209. $this->logMessage('Importing Taxonomy IDs...');
  210. foreach ($tax_ids as $tax_id) {
  211. $tax_id = trim($tax_id);
  212. $this->importRecord($tax_id);
  213. $this->addItemsHandled(1);
  214. }
  215. }
  216. // If the user wants to update existing records then do that.
  217. if ($import_existing) {
  218. $this->logMessage('Updating Existing...');
  219. $this->updateExisting();
  220. }
  221. // Now import the tree.
  222. $options = ['taxonomy' => 1];
  223. chado_phylogeny_import_tree($this->tree, $this->phylotree, $options);
  224. }
  225. /**
  226. * Create the taxonomic tree in Chado.
  227. *
  228. * If the tree already exists it will not be recreated.
  229. *
  230. * @throws Exception
  231. * @return
  232. * Returns the phylotree object.
  233. */
  234. private function initTree() {
  235. // Add the taxonomy tree record into the phylotree table. If the tree
  236. // already exists then don't insert it again.
  237. $site_name = variable_get('site_name');
  238. $tree_name = $site_name . 'Taxonomy Tree';
  239. $phylotree = chado_select_record('phylotree', ['*'], ['name' => $tree_name]);
  240. if (count($phylotree) == 0) {
  241. // Add the taxonomic tree.
  242. $phylotree = [
  243. 'name' => $site_name . 'Taxonomy Tree',
  244. 'description' => 'A phylogenetic tree based on taxonomic rank.',
  245. 'leaf_type' => 'taxonomy',
  246. 'tree_file' => '/dev/null',
  247. 'format' => 'taxonomy',
  248. 'no_load' => TRUE,
  249. ];
  250. $errors = [];
  251. $warnings = [];
  252. $success = tripal_insert_phylotree($phylotree, $errors, $warnings);
  253. if (!$success) {
  254. throw new Exception("Cannot add the Taxonomy Tree record.");
  255. }
  256. $phylotree = (object) $phylotree;
  257. }
  258. else {
  259. $phylotree = $phylotree[0];
  260. }
  261. return $phylotree;
  262. }
  263. /**
  264. * Iterates through all existing organisms and rebuilds the taxonomy tree.
  265. *
  266. * The phloytree API doesn't support adding nodes to existing trees only
  267. * importing whole trees. So, we must rebuild the tree using the current
  268. * organisms and then we can add to it.
  269. *
  270. */
  271. private function rebuildTree() {
  272. $lineage_nodes[] = [];
  273. // Get the "rank" cvterm. It requires that the TAXRANK vocabulary is loaded.
  274. $rank_cvterm = chado_get_cvterm([
  275. 'name' => 'rank',
  276. 'cv_id' => ['name' => 'local'],
  277. ]);
  278. // The taxonomic tree must have a root, so create that first.
  279. $tree = [
  280. 'name' => 'root',
  281. 'depth' => 0,
  282. 'is_root' => 1,
  283. 'is_leaf' => 0,
  284. 'is_internal' => 0,
  285. 'left_index' => 0,
  286. 'right_index' => 0,
  287. 'branch_set' => [],
  288. ];
  289. $total = count($this->all_orgs);
  290. $j = 1;
  291. foreach ($this->all_orgs as $organism) {
  292. $sci_name = chado_get_organism_scientific_name($organism);
  293. //$this->logMessage("- " . ($j++) . " of $total. Adding @organism", array('@organism' => $sci_name));
  294. // First get the phylonode record for this organism.
  295. $sql = "
  296. SELECT P.*
  297. FROM {phylonode} P
  298. INNER JOIN {phylonode_organism} PO on PO.phylonode_id = P.phylonode_id
  299. WHERE P.phylotree_id = :phylotree_id AND PO.organism_id = :organism_id
  300. ";
  301. $args = [
  302. ':phylotree_id' => $this->phylotree->phylotree_id,
  303. ':organism_id' => $organism->organism_id,
  304. ];
  305. $result = chado_query($sql, $args);
  306. if (!$result) {
  307. continue;
  308. }
  309. $phylonode = $result->fetchObject();
  310. // Next get the lineage for this organism.
  311. $lineage = $this->getProperty($organism->organism_id, 'lineage');
  312. if (!$lineage) {
  313. continue;
  314. }
  315. $lineage_depth = preg_split('/;\s*/', $lineage->value);
  316. // Now rebuild the tree by first creating the nodes for the full
  317. // lineage and then adding the organism as a leaf node.
  318. $parent = $tree;
  319. $i = 1;
  320. $lineage_good = TRUE;
  321. foreach ($lineage_depth as $child) {
  322. // We need to find the node in the phylotree for this level of the
  323. // lineage, but there's a lot of repeats and we don't want to keep
  324. // doing the same queries over and over, so we store the nodes
  325. // we've already seen in the $lineage_nodes array for fast lookup.
  326. if (array_key_exists($child, $lineage_nodes)) {
  327. $phylonode = $lineage_nodes[$child];
  328. if (!$phylonode) {
  329. $lineage_good = FALSE;
  330. continue;
  331. }
  332. }
  333. else {
  334. $values = [
  335. 'phylotree_id' => $this->phylotree->phylotree_id,
  336. 'label' => $child,
  337. ];
  338. $columns = ['*'];
  339. $phylonode = chado_select_record('phylonode', $columns, $values);
  340. if (count($phylonode) == 0) {
  341. $lineage_nodes[$child] = NULL;
  342. $lineage_good = FALSE;
  343. continue;
  344. }
  345. $phylonode = $phylonode[0];
  346. $lineage_nodes[$child] = $phylonode;
  347. $values = [
  348. 'phylonode_id' => $phylonode->phylonode_id,
  349. 'type_id' => $rank_cvterm->cvterm_id,
  350. ];
  351. $columns = ['*'];
  352. $phylonodeprop = chado_select_record('phylonodeprop', $columns, $values);
  353. }
  354. $name = $child;
  355. $node_rank = (string) $child->Rank;
  356. $node = [
  357. 'name' => $name,
  358. 'depth' => $i,
  359. 'is_root' => 0,
  360. 'is_leaf' => 0,
  361. 'is_internal' => 1,
  362. 'left_index' => 0,
  363. 'right_index' => 0,
  364. 'parent' => $parent,
  365. 'branch_set' => [],
  366. 'parent' => $parent['name'],
  367. 'properties' => [
  368. $rank_cvterm->cvterm_id => $phylonodeprop[0]->value,
  369. ],
  370. ];
  371. $parent = $node;
  372. $this->addTaxonomyNode($tree, $node, $lineage_depth);
  373. $i++;
  374. } // end foreach ($lineage_depth as $child) { ...
  375. // If $stop is set then we had problems setting the lineage so
  376. // skip adding the leaf node below.
  377. if (!$lineage_good) {
  378. continue;
  379. }
  380. $rank_type = 'species';
  381. if (property_exists($organism, 'type_id') and $organism->type_id) {
  382. $rank_type = $organism->type;
  383. }
  384. // Now add in the leaf node
  385. $sci_name = chado_get_organism_scientific_name($organism);
  386. $node = [
  387. 'name' => $sci_name,
  388. 'depth' => $i,
  389. 'is_root' => 0,
  390. 'is_leaf' => 1,
  391. 'is_internal' => 0,
  392. 'left_index' => 0,
  393. 'right_index' => 0,
  394. 'parent' => $parent['name'],
  395. 'organism_id' => $organism->organism_id,
  396. 'properties' => [
  397. $rank_cvterm->cvterm_id => $rank_type,
  398. ],
  399. ];
  400. $this->addTaxonomyNode($tree, $node, $lineage_depth);
  401. // Set the indecies for the tree.
  402. chado_assign_phylogeny_tree_indices($tree);
  403. }
  404. return $tree;
  405. }
  406. /**
  407. * Imports details from NCBI Taxonomy for organisms that alrady exist.
  408. */
  409. private function updateExisting() {
  410. $i = 0;
  411. $total = count($this->all_orgs);
  412. foreach ($this->all_orgs as $organism) {
  413. // If the organism record is marked as new then let's skip it because
  414. // it was newly added and should have the updated information already.
  415. if ($organism->is_new) {
  416. continue;
  417. }
  418. // TODO: we should check if the organism already has a taxonomy ID.
  419. // if so we should use that instead of the scientific name.
  420. // Build the query string to get the information about this species.
  421. $sci_name = chado_get_organism_scientific_name($organism);
  422. $sci_name = urlencode($sci_name);
  423. $search_url = "http://www.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?" .
  424. "db=taxonomy" .
  425. "&term=$sci_name";
  426. // Get the search response from NCBI.
  427. $rfh = fopen($search_url, "r");
  428. $xml_text = '';
  429. if (!$rfh) {
  430. $this->logMessage("Could not look up !sci_name", ['!sci_name' => $sci_name], TRIPAL_WARNING);
  431. continue;
  432. }
  433. while (!feof($rfh)) {
  434. $xml_text .= fread($rfh, 255);
  435. }
  436. fclose($rfh);
  437. // Parse the XML to get the taxonomy ID
  438. $xml = new SimpleXMLElement($xml_text);
  439. if ($xml) {
  440. $taxid = (string) $xml->IdList->Id;
  441. if ($taxid) {
  442. $this->importRecord($taxid, $organism);
  443. }
  444. }
  445. $this->addItemsHandled(1);
  446. // NCBI limits requests to 3/second.
  447. if ($i % 3 == 0) {
  448. sleep(1);
  449. }
  450. $i++;
  451. }
  452. }
  453. /**
  454. * Checks the Chado database to see if the organism already exists.
  455. *
  456. * @param $taxid
  457. * The taxonomic ID for the organism.
  458. * @param $sci_name
  459. * The scientific name for the organism as returned by NCBI
  460. */
  461. private function findOrganism($taxid, $sci_name) {
  462. $organism = NULL;
  463. // First check the taxid to see if it's present and assocaited with an
  464. // organism already.
  465. $values = [
  466. 'db_id' => [
  467. 'name' => 'NCBITaxon',
  468. ],
  469. 'accession' => $taxid,
  470. ];
  471. $columns = ['dbxref_id'];
  472. $dbxref = chado_select_record('dbxref', $columns, $values);
  473. if (count($dbxref) > 0) {
  474. $columns = ['organism_id'];
  475. $values = ['dbxref_id' => $dbxref[0]->dbxref_id];
  476. $organism_dbxref = chado_select_record('organism_dbxref', $columns, $values);
  477. if (count($organism_dbxref) > 0) {
  478. $organism_id = $organism_dbxref[0]->organism_id;
  479. $columns = ['*'];
  480. $values = ['organism_id' => $organism_id];
  481. $organism = chado_select_record('organism', $columns, $values);
  482. if (count($organism) > 0) {
  483. $organism = $organism[0];
  484. }
  485. }
  486. }
  487. // If the caller did not provide an organism then we want to try and
  488. // add one. But, it only makes sense to add one if this record
  489. // is of rank species.
  490. // First check if the full name (including the infrasepcific name)
  491. // are all present in the genus and species name. This would have
  492. // been the Chado v1.2 (or less) of storing species.
  493. if (!$organism) {
  494. $sql = "
  495. SELECT organism_id
  496. FROM {organism}
  497. WHERE concat(genus, ' ', species) = :sci_name
  498. ";
  499. $results = chado_query($sql, [':sci_name' => $sci_name]);
  500. $item = $results->fetchObject();
  501. if ($item) {
  502. $columns = ['*'];
  503. $values = ['organism_id' => $item->organism_id];
  504. $organism = chado_select_record('organism', $columns, $values);
  505. if (count($organism) > 0) {
  506. $organism = $organism[0];
  507. }
  508. }
  509. }
  510. // Second, check if the full name includes the infraspecific name.
  511. if (!$organism) {
  512. foreach ($this->all_orgs as $item) {
  513. $internal_sci_name = chado_get_organism_scientific_name($item);
  514. if ($sci_name == $internal_sci_name) {
  515. $organism = $item;
  516. }
  517. }
  518. }
  519. return $organism;
  520. }
  521. /**
  522. * Adds a new organism record to Chado.
  523. *
  524. * @param sci_name
  525. * The scientific name as provied by NCBI Taxonomy.
  526. * @param $rank
  527. * The rank of the organism as provied by NCBI Taxonomy.
  528. */
  529. private function addOrganism($sci_name, $rank) {
  530. $organism = NULL;
  531. $matches = [];
  532. $genus = '';
  533. $species = '';
  534. $infra = '';
  535. $values = [];
  536. // Check if the scientific name has an infraspecific part or is just
  537. // a species name.
  538. if (preg_match('/^(.+?)\s+(.+?)\s+(.+)$/', $sci_name, $matches)) {
  539. $genus = $matches[1];
  540. $species = $matches[2];
  541. $infra = $matches[3];
  542. // Get the CV term for the rank.
  543. $type = chado_get_cvterm([
  544. 'name' => preg_replace('/ /', '_', $rank),
  545. 'cv_id' => ['name' => 'taxonomic_rank'],
  546. ]);
  547. // Remove the rank from the infraspecific name.
  548. $abbrev = chado_abbreviate_infraspecific_rank($rank);
  549. $infra = preg_replace("/$abbrev/", "", $infra);
  550. $infra = trim($infra);
  551. $values = [
  552. 'genus' => $genus,
  553. 'species' => $species,
  554. 'abbreviation' => $genus[0] . '. ' . $species,
  555. 'type_id' => $type->cvterm_id,
  556. 'infraspecific_name' => $infra,
  557. ];
  558. $organism = chado_insert_record('organism', $values);
  559. $organism = (object) $organism;
  560. $organism->type = $rank;
  561. }
  562. else {
  563. if (preg_match('/^(.+?)\s+(.+?)$/', $sci_name, $matches)) {
  564. $genus = $matches[1];
  565. $species = $matches[2];
  566. $infra = '';
  567. $values = [
  568. 'genus' => $genus,
  569. 'species' => $species,
  570. 'abbreviation' => $genus[0] . '. ' . $species,
  571. ];
  572. $organism = chado_insert_record('organism', $values);
  573. $organism = (object) $organism;
  574. }
  575. }
  576. if ($organism) {
  577. $organism->is_new = TRUE;
  578. $this->all_orgs[] = $organism;
  579. }
  580. return $organism;
  581. }
  582. /**
  583. * Imports an organism from the NCBI taxonomy DB by its taxonomy ID
  584. *
  585. * @param $taxid
  586. * The NCBI Taxonomy ID.
  587. * @param $organism
  588. * The organism object to which this taxonomy belongs. If the organism
  589. * is NULL then it will be created.
  590. */
  591. private function importRecord($taxid, $organism = NULL) {
  592. $adds_organism = $organism ? FALSE : TRUE;
  593. // Get the "rank" cvterm. It requires that the TAXRANK vocabulary is loaded.
  594. $rank_cvterm = chado_get_cvterm([
  595. 'name' => 'rank',
  596. 'cv_id' => ['name' => 'local'],
  597. ]);
  598. // Get the details for this taxonomy.
  599. $fetch_url = "http://www.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?" .
  600. "db=taxonomy" .
  601. "&id=$taxid";
  602. // Get the search response from NCBI.
  603. $rfh = fopen($fetch_url, "r");
  604. $xml_text = '';
  605. while (!feof($rfh)) {
  606. $xml_text .= fread($rfh, 255);
  607. }
  608. fclose($rfh);
  609. $xml = new SimpleXMLElement($xml_text);
  610. if ($xml) {
  611. $taxon = $xml->Taxon;
  612. // Get the genus and species from the xml.
  613. $parent = (string) $taxon->ParentTaxId;
  614. $rank = (string) $taxon->Rank;
  615. $sci_name = (string) $taxon->ScientificName;
  616. //$this->logMessage(' - Importing @sci_name', array('@sci_name' => $sci_name));
  617. // If we don't have an organism record provided then see if there
  618. // is one provided by Chado, if not, the try to add one.
  619. if (!$organism) {
  620. $organism = $this->findOrganism($taxid, $sci_name);
  621. if (!$organism) {
  622. $organism = $this->addOrganism($sci_name, $rank);
  623. if (!$organism) {
  624. throw new Exception(t('Cannot add organism: @sci_name', ['@sci_name' => $sci_name]));
  625. }
  626. }
  627. }
  628. // Associate the Dbxref with the organism.
  629. $this->addDbxref($organism->organism_id, $taxid);
  630. // Get properties for this organism.
  631. $lineage = (string) $taxon->Lineage;
  632. $genetic_code = (string) $taxon->GeneticCode->GCId;
  633. $genetic_code_name = (string) $taxon->GeneticCode->GCName;
  634. $mito_genetic_code = (string) $taxon->MitoGeneticCode->MGCId;
  635. $mito_genetic_code_name = (string) $taxon->MitoGeneticCode->MGCName;
  636. $division = (string) $taxon->Division;
  637. // Add in the organism properties.
  638. $this->addProperty($organism->organism_id, 'division', $division);
  639. $this->addProperty($organism->organism_id, 'mitochondrial_genetic_code_name', $mito_genetic_code_name);
  640. $this->addProperty($organism->organism_id, 'mitochondrial_genetic_code', $mito_genetic_code);
  641. $this->addProperty($organism->organism_id, 'genetic_code_name', $genetic_code_name);
  642. $this->addProperty($organism->organism_id, 'lineage', $lineage);
  643. $this->addProperty($organism->organism_id, 'genetic_code', $genetic_code);
  644. $name_ranks = [];
  645. if ($taxon->OtherNames->children) {
  646. foreach ($taxon->OtherNames->children() as $child) {
  647. $type = $child->getName();
  648. $name = (string) $child;
  649. if (!array_key_exists($type, $name_ranks)) {
  650. $name_ranks[$type] = 0;
  651. }
  652. switch ($type) {
  653. case 'GenbankCommonName':
  654. $this->addProperty($organism->organism_id, 'genbank_common_name', $name, $name_ranks[$type]);
  655. break;
  656. case 'Synonym':
  657. case 'GenbankSynonym':
  658. $this->addProperty($organism->organism_id, 'synonym', $name, $name_ranks[$type]);
  659. break;
  660. case 'CommonName':
  661. // If we had to add the organism then include the commone name too.
  662. if ($adds_organism) {
  663. $organism->common_name = $name;
  664. $values = ['organism_id' => $organism->id];
  665. chado_update_record('organism', $values, $organism);
  666. }
  667. case 'Includes':
  668. $this->addProperty($organism->organism_id, 'other_name', $name, $name_ranks[$type]);
  669. break;
  670. case 'EquivalentName':
  671. $this->addProperty($organism->organism_id, 'equivalent_name', $name, $name_ranks[$type]);
  672. break;
  673. case 'Anamorph':
  674. $this->addProperty($organism->organism_id, 'anamorph', $name, $name_ranks[$type]);
  675. break;
  676. case 'Name':
  677. // skip the Name stanza
  678. break;
  679. default:
  680. print "NOTICE: Skipping unrecognzed name type: $type\n";
  681. // do nothing for unrecognized types
  682. }
  683. $name_ranks[$type]++;
  684. }
  685. }
  686. // Generate a nested array structure that can be used for importing the tree.
  687. $lineage_depth = preg_split('/;\s*/', $lineage);
  688. $parent = $this->tree;
  689. $i = 1;
  690. foreach ($taxon->LineageEx->children() as $child) {
  691. $tid = (string) $child->TaxID;
  692. $name = (string) $child->ScientificName;
  693. $node_rank = (string) $child->Rank;
  694. $node = [
  695. 'name' => $name,
  696. 'depth' => $i,
  697. 'is_root' => 0,
  698. 'is_leaf' => 0,
  699. 'is_internal' => 1,
  700. 'left_index' => 0,
  701. 'right_index' => 0,
  702. 'parent' => $parent,
  703. 'branch_set' => [],
  704. 'parent' => $parent['name'],
  705. 'properties' => [
  706. $rank_cvterm->cvterm_id => $node_rank,
  707. ],
  708. ];
  709. $parent = $node;
  710. $this->addTaxonomyNode($this->tree, $node, $lineage_depth);
  711. $i++;
  712. }
  713. // Now add in the leaf node
  714. $node = [
  715. 'name' => $sci_name,
  716. 'depth' => $i,
  717. 'is_root' => 0,
  718. 'is_leaf' => 1,
  719. 'is_internal' => 0,
  720. 'left_index' => 0,
  721. 'right_index' => 0,
  722. 'parent' => $parent['name'],
  723. 'organism_id' => $organism->organism_id,
  724. 'properties' => [
  725. $rank_cvterm->cvterm_id => $rank,
  726. ],
  727. ];
  728. $this->addTaxonomyNode($this->tree, $node, $lineage_depth);
  729. // Set the indecies for the tree.
  730. chado_assign_phylogeny_tree_indices($this->tree);
  731. }
  732. }
  733. /**
  734. *
  735. */
  736. private function addTaxonomyNode(&$tree, $node, $lineage_depth) {
  737. // Get the branch set for the tree root.
  738. $branch_set = &$tree['branch_set'];
  739. // Iterate through the tree up until the depth where this node will
  740. // be placed.
  741. $node_depth = $node['depth'];
  742. for ($i = 1; $i <= $node_depth; $i++) {
  743. // Iterate through any existing nodes in the branch set to see if
  744. // the node name matches the correct name for the lineage at this
  745. // depth. If it matches then it is inside of this branch set that
  746. // we will place the node.
  747. for ($j = 0; $j < count($branch_set); $j++) {
  748. // If this node already exists in the tree then return.
  749. if ($branch_set[$j]['name'] == $node['name'] and
  750. $branch_set[$j]['depth'] = $node['depth']) {
  751. return;
  752. }
  753. // Otherwise, set the branch to be the current branch and continue.
  754. if ($branch_set[$j]['name'] == $lineage_depth[$i - 1]) {
  755. $branch_set = &$branch_set[$j]['branch_set'];
  756. break;
  757. }
  758. }
  759. }
  760. // Add the node to the last branch set. This should be where this node goes.
  761. $branch_set[] = $node;
  762. }
  763. /**
  764. * Retrieves a property for a given organism.
  765. *
  766. * @param $organism_id
  767. * The organism ID to which the property is added.
  768. * @param $term_name
  769. * The name of the organism property term. This term must be
  770. * present in the 'organism_property' cv.
  771. * @param $rank
  772. * The order for this property. The first instance of this term for
  773. * this organism should be zero. Defaults to zero.
  774. *
  775. * @return
  776. * The property object.
  777. */
  778. private function getProperty($organism_id, $term_name, $rank = 0) {
  779. $record = [
  780. 'table' => 'organism',
  781. 'id' => $organism_id,
  782. ];
  783. $property = [
  784. 'type_name' => $term_name,
  785. 'cv_name' => 'organism_property',
  786. 'value' => $value,
  787. 'rank' => $rank,
  788. ];
  789. return chado_get_property($record, $property);
  790. }
  791. /**
  792. * Adds a property to an organism node.
  793. *
  794. * @param $organism_id
  795. * The organism ID to which the property is added.
  796. * @param $term_name
  797. * The name of the organism property term. This term must be
  798. * present in the 'organism_property' cv.
  799. * @param $value
  800. * The value of the property.
  801. * @param $rank
  802. * The order for this property. The first instance of this term for
  803. * this organism should be zero. Defaults to zero.
  804. */
  805. private function addProperty($organism_id, $term_name, $value, $rank = 0) {
  806. if (!$value) {
  807. return;
  808. }
  809. $record = [
  810. 'table' => 'organism',
  811. 'id' => $organism_id,
  812. ];
  813. $property = [
  814. 'type_name' => $term_name,
  815. 'cv_name' => 'organism_property',
  816. 'value' => $value,
  817. ];
  818. // Delete all properties of this type if the rank is zero.
  819. if ($rank == 0) {
  820. chado_delete_property($record, $property);
  821. }
  822. chado_insert_property($record, $property);
  823. }
  824. /**
  825. *
  826. * @param unknown $organism_id
  827. * @param unknown $taxId
  828. */
  829. private function addDbxref($organism_id, $taxId) {
  830. $db = chado_get_db(['name' => 'NCBITaxon']);
  831. $values = [
  832. 'db_id' => $db->db_id,
  833. 'accession' => $taxId,
  834. ];
  835. $dbxref = chado_insert_dbxref($values);
  836. $values = [
  837. 'dbxref_id' => $dbxref->dbxref_id,
  838. 'organism_id' => $organism_id,
  839. ];
  840. if (!chado_select_record('organism_dbxref', ['organism_dbxref_id'], $values)) {
  841. chado_insert_record('organism_dbxref', $values);
  842. }
  843. }
  844. }