FASTAImporter.inc 36 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009
  1. <?php
  2. class FASTAImporter extends TripalImporter {
  3. /**
  4. * The name of this loader. This name will be presented to the site
  5. * user.
  6. */
  7. public static $name = 'Chado FASTA Loader';
  8. /**
  9. * The machine name for this loader. This name will be used to construct
  10. * the URL for the loader.
  11. */
  12. public static $machine_name = 'chado_fasta_loader';
  13. /**
  14. * A brief description for this loader. This description will be
  15. * presented to the site user.
  16. */
  17. public static $description = 'Load sequences from a multi-FASTA file into Chado';
  18. /**
  19. * An array containing the extensions of allowed file types.
  20. */
  21. public static $file_types = [
  22. 'fasta',
  23. 'txt',
  24. 'fa',
  25. 'aa',
  26. 'pep',
  27. 'nuc',
  28. 'faa',
  29. 'fna',
  30. ];
  31. /**
  32. * Provides information to the user about the file upload. Typically this
  33. * may include a description of the file types allowed.
  34. */
  35. public static $upload_description = 'Please provide the FASTA file. The file must have a .fasta extension.';
  36. /**
  37. * The title that should appear above the file upload section.
  38. */
  39. public static $upload_title = 'FASTA Upload';
  40. /**
  41. * Text that should appear on the button at the bottom of the importer
  42. * form.
  43. */
  44. public static $button_text = 'Import FASTA file';
  45. /**
  46. * Indicates the methods that the file uploader will support.
  47. */
  48. public static $methods = [
  49. // Allow the user to upload a file to the server.
  50. 'file_upload' => TRUE,
  51. // Allow the user to provide the path on the Tripal server for the file.
  52. 'file_local' => TRUE,
  53. // Allow the user to provide a remote URL for the file.
  54. 'file_remote' => TRUE,
  55. ];
  56. /**
  57. * @see TripalImporter::form()
  58. */
  59. public function form($form, &$form_state) {
  60. // get the list of organisms
  61. $sql = "SELECT * FROM {organism} ORDER BY genus, species";
  62. $org_rset = chado_query($sql);
  63. $organisms = [];
  64. $organisms[''] = '';
  65. while ($organism = $org_rset->fetchObject()) {
  66. $organisms[$organism->organism_id] = "$organism->genus $organism->species ($organism->common_name)";
  67. }
  68. $form['organism_id'] = [
  69. '#title' => t('Organism'),
  70. '#type' => t('select'),
  71. '#description' => t("Choose the organism to which these sequences are associated"),
  72. '#required' => TRUE,
  73. '#options' => $organisms,
  74. ];
  75. // get the sequence ontology CV ID
  76. $values = ['name' => 'sequence'];
  77. $cv = chado_select_record('cv', ['cv_id'], $values);
  78. $cv_id = $cv[0]->cv_id;
  79. $form['seqtype'] = [
  80. '#type' => 'textfield',
  81. '#title' => t('Sequence Type'),
  82. '#required' => TRUE,
  83. '#description' => t('Please enter the Sequence Ontology (SO) term name that describes the sequences in the FASTA file (e.g. gene, mRNA, polypeptide, etc...)'),
  84. '#autocomplete_path' => "admin/tripal/storage/chado/auto_name/cvterm/$cv_id",
  85. ];
  86. $form['method'] = [
  87. '#type' => 'radios',
  88. '#title' => 'Method',
  89. '#required' => TRUE,
  90. '#options' => [
  91. t('Insert only'),
  92. t('Update only'),
  93. t('Insert and update'),
  94. ],
  95. '#description' => t('Select how features in the FASTA file are handled.
  96. Select "Insert only" to insert the new features. If a feature already
  97. exists with the same name or unique name and type then it is skipped.
  98. Select "Update only" to only update featues that already exist in the
  99. database. Select "Insert and Update" to insert features that do
  100. not exist and upate those that do.'),
  101. '#default_value' => 2,
  102. ];
  103. $form['match_type'] = [
  104. '#type' => 'radios',
  105. '#title' => 'Name Match Type',
  106. '#required' => TRUE,
  107. '#options' => [
  108. t('Name'),
  109. t('Unique name'),
  110. ],
  111. '#description' => t('Used for "updates only" or "insert and update" methods. Not required if method type is "insert".
  112. Feature data is stored in Chado with both a human-readable
  113. name and a unique name. If the features in your FASTA file are uniquely identified using
  114. a human-readable name then select the "Name" button. If your features are
  115. uniquely identified using the unique name then select the "Unique name" button. If you
  116. loaded your features first using the GFF loader then the unique name of each
  117. features were indicated by the "ID=" attribute and the name by the "Name=" attribute.
  118. By default, the FASTA loader will use the first word (character string
  119. before the first space) as the name for your feature. If
  120. this does not uniquely identify your feature consider specifying a regular expression in the advanced section below.
  121. Additionally, you may import both a name and a unique name for each sequence using the advanced options.'),
  122. '#default_value' => 1,
  123. ];
  124. // Additional Options
  125. $form['additional'] = [
  126. '#type' => 'fieldset',
  127. '#title' => t('Additional Options'),
  128. '#collapsible' => TRUE,
  129. '#collapsed' => TRUE,
  130. ];
  131. $form['additional']['re_help'] = [
  132. '#type' => 'item',
  133. '#value' => t('A regular expression is an advanced method for extracting
  134. information from a string of text. Your FASTA file may contain both a
  135. human-readable name and a unique name for each sequence. If you want
  136. to import both the name and unique name for all sequences, then you
  137. must provide regular expressions so that the loader knows how to
  138. separate them. Otherwise the name and uniquename will be the same.
  139. By default, this loader will use the first word in the definition
  140. lines of the FASTA file
  141. as the name or unique name of the feature.'),
  142. ];
  143. $form['additional']['re_name'] = [
  144. '#type' => 'textfield',
  145. '#title' => t('Regular expression for the name'),
  146. '#required' => FALSE,
  147. '#description' => t('Enter the regular expression that will extract the
  148. feature name from the FASTA definition line. For example, for a
  149. defintion line with a name and unique name separated by a bar \'|\' (>seqname|uniquename),
  150. the regular expression for the name would be, "^(.*?)\|.*$". All FASTA
  151. definition lines begin with the ">" symbol. You do not need to incldue
  152. this symbol in your regular expression.'),
  153. ];
  154. $form['additional']['re_uname'] = [
  155. '#type' => 'textfield',
  156. '#title' => t('Regular expression for the unique name'),
  157. '#required' => FALSE,
  158. '#description' => t('Enter the regular expression that will extract the
  159. feature name from the FASTA definition line. For example, for a
  160. defintion line with a name and unique name separated by a bar \'|\' (>seqname|uniquename),
  161. the regular expression for the unique name would be "^.*?\|(.*)$". All FASTA
  162. definition lines begin with the ">" symbol. You do not need to incldue
  163. this symbol in your regular expression.'),
  164. ];
  165. // Advanced database cross reference options.
  166. $form['additional']['db'] = [
  167. '#type' => 'fieldset',
  168. '#title' => t('External Database Reference'),
  169. '#weight' => 6,
  170. '#collapsed' => TRUE,
  171. ];
  172. $form['additional']['db']['re_accession'] = [
  173. '#type' => 'textfield',
  174. '#title' => t('Regular expression for the accession'),
  175. '#required' => FALSE,
  176. '#description' => t('Enter the regular expression that will extract the accession for the external database for each feature from the FASTA definition line.'),
  177. '#weight' => 2,
  178. ];
  179. // get the list of databases
  180. $sql = "SELECT * FROM {db} ORDER BY name";
  181. $db_rset = chado_query($sql);
  182. $dbs = [];
  183. $dbs[''] = '';
  184. while ($db = $db_rset->fetchObject()) {
  185. $dbs[$db->db_id] = "$db->name";
  186. }
  187. $form['additional']['db']['db_id'] = [
  188. '#title' => t('External Database'),
  189. '#type' => t('select'),
  190. '#description' => t("Plese choose an external database for which these sequences have a cross reference."),
  191. '#required' => FALSE,
  192. '#options' => $dbs,
  193. '#weight' => 1,
  194. ];
  195. $form['additional']['relationship'] = [
  196. '#type' => 'fieldset',
  197. '#title' => t('Relationships'),
  198. '#weight' => 6,
  199. '#collapsed' => TRUE,
  200. ];
  201. $rels = [];
  202. $rels[''] = '';
  203. $rels['part_of'] = 'part of';
  204. $rels['derives_from'] = 'produced by (derives from)';
  205. // Advanced references options
  206. $form['additional']['relationship']['rel_type'] = [
  207. '#title' => t('Relationship Type'),
  208. '#type' => t('select'),
  209. '#description' => t("Use this option to create associations, or relationships between the
  210. features of this FASTA file and existing features in the database. For
  211. example, to associate a FASTA file of peptides to existing genes or transcript sequence,
  212. select the type 'produced by'. For a CDS sequences select the type 'part of'"),
  213. '#required' => FALSE,
  214. '#options' => $rels,
  215. '#weight' => 5,
  216. ];
  217. $form['additional']['relationship']['re_subject'] = [
  218. '#type' => 'textfield',
  219. '#title' => t('Regular expression for the parent'),
  220. '#required' => FALSE,
  221. '#description' => t('Enter the regular expression that will extract the unique
  222. name needed to identify the existing sequence for which the
  223. relationship type selected above will apply. If no regular
  224. expression is provided, the parent unique name must be the
  225. same as the loaded feature name.'),
  226. '#weight' => 6,
  227. ];
  228. $form['additional']['relationship']['parent_type'] = [
  229. '#type' => 'textfield',
  230. '#title' => t('Parent Type'),
  231. '#required' => FALSE,
  232. '#description' => t('Please enter the Sequence Ontology term for the parent. For example
  233. if the FASTA file being loaded is a set of proteins that are
  234. products of genes, then use the SO term \'gene\' or \'transcript\' or equivalent. However,
  235. this type must match the type for already loaded features.'),
  236. '#weight' => 7,
  237. ];
  238. return $form;
  239. }
  240. /**
  241. * @see TripalImporter::formValidate()
  242. */
  243. public function formValidate($form, &$form_state) {
  244. $organism_id = $form_state['values']['organism_id'];
  245. $type = trim($form_state['values']['seqtype']);
  246. $method = trim($form_state['values']['method']);
  247. $match_type = trim($form_state['values']['match_type']);
  248. $re_name = trim($form_state['values']['re_name']);
  249. $re_uname = trim($form_state['values']['re_uname']);
  250. $re_accession = trim($form_state['values']['re_accession']);
  251. $db_id = $form_state['values']['db_id'];
  252. $rel_type = $form_state['values']['rel_type'];
  253. $re_subject = trim($form_state['values']['re_subject']);
  254. $parent_type = trim($form_state['values']['parent_type']);
  255. if ($method == 0) {
  256. $method = 'Insert only';
  257. }
  258. if ($method == 1) {
  259. $method = 'Update only';
  260. }
  261. if ($method == 2) {
  262. $method = 'Insert and update';
  263. }
  264. if ($match_type == 0) {
  265. $match_type = 'Name';
  266. }
  267. if ($match_type == 1) {
  268. $match_type = 'Unique name';
  269. }
  270. if ($re_name and !$re_uname and strcmp($match_type, 'Unique name') == 0) {
  271. form_set_error('re_uname', t("You must provide a regular expression to identify the sequence unique name"));
  272. }
  273. if (!$re_name and $re_uname and strcmp($match_type, 'Name') == 0) {
  274. form_set_error('re_name', t("You must provide a regular expression to identify the sequence name"));
  275. }
  276. // make sure if a relationship is specified that all fields are provided.
  277. if (($rel_type or $re_subject) and !$parent_type) {
  278. form_set_error('parent_type', t("Please provide a SO term for the parent"));
  279. }
  280. if (($parent_type or $re_subject) and !$rel_type) {
  281. form_set_error('rel_type', t("Please select a relationship type"));
  282. }
  283. // make sure if a database is specified that all fields are provided
  284. if ($db_id and !$re_accession) {
  285. form_set_error('re_accession', t("Please provide a regular expression for the accession"));
  286. }
  287. if ($re_accession and !$db_id) {
  288. form_set_error('db_id', t("Please select a database"));
  289. }
  290. // check to make sure the types exists
  291. $cvtermsql = "
  292. SELECT CVT.cvterm_id
  293. FROM {cvterm} CVT
  294. INNER JOIN {cv} CV on CVT.cv_id = CV.cv_id
  295. LEFT JOIN {cvtermsynonym} CVTS on CVTS.cvterm_id = CVT.cvterm_id
  296. WHERE cv.name = :cvname and (CVT.name = :name or CVTS.synonym = :synonym)
  297. ";
  298. $cvterm = chado_query($cvtermsql,
  299. [
  300. ':cvname' => 'sequence',
  301. ':name' => $type,
  302. ':synonym' => $type,
  303. ])->fetchObject();
  304. if (!$cvterm) {
  305. form_set_error('type', t("The Sequence Ontology (SO) term selected for the sequence type is not available in the database. Please check spelling or select another."));
  306. }
  307. if ($rel_type) {
  308. $cvterm = chado_query($cvtermsql, [
  309. ':cvname' => 'sequence',
  310. ':name' => $parent_type,
  311. ':synonym' => $parent_type,
  312. ])->fetchObject();
  313. if (!$cvterm) {
  314. form_set_error('parent_type', t("The Sequence Ontology (SO) term selected for the parent relationship is not available in the database. Please check spelling or select another."));
  315. }
  316. }
  317. }
  318. /**
  319. * @see TripalImporter::run()
  320. */
  321. public function run() {
  322. $arguments = $this->arguments['run_args'];
  323. $file_path = $this->arguments['files'][0]['file_path'];
  324. $organism_id = $arguments['organism_id'];
  325. $type = $arguments['seqtype'];
  326. $method = $arguments['method'];
  327. $match_type = $arguments['match_type'];
  328. $re_name = $arguments['re_name'];
  329. $re_uname = $arguments['re_uname'];
  330. $re_accession = $arguments['re_accession'];
  331. $db_id = $arguments['db_id'];
  332. $rel_type = $arguments['rel_type'];
  333. $re_subject = $arguments['re_subject'];
  334. $parent_type = $arguments['parent_type'];
  335. $method = $arguments['method'];
  336. $analysis_id = $arguments['analysis_id'];
  337. $match_type = $arguments['match_type'];
  338. if ($method == 0) {
  339. $method = 'Insert only';
  340. }
  341. if ($method == 1) {
  342. $method = 'Update only';
  343. }
  344. if ($method == 2) {
  345. $method = 'Insert and update';
  346. }
  347. if ($match_type == 0) {
  348. $match_type = 'Name';
  349. }
  350. if ($match_type == 1) {
  351. $match_type = 'Unique name';
  352. }
  353. $this->loadFasta($file_path, $organism_id, $type, $re_name, $re_uname, $re_accession,
  354. $db_id, $rel_type, $re_subject, $parent_type, $method, $analysis_id,
  355. $match_type);
  356. }
  357. /**
  358. * Load a fasta file.
  359. *
  360. * @param $file_path
  361. * The full path to the fasta file to load.
  362. * @param $organism_id
  363. * The organism_id of the organism these features are from.
  364. * @param $type
  365. * The type of features contained in the fasta file.
  366. * @param $re_name
  367. * The regular expression to extract the feature.name from the fasta header.
  368. * @param $re_uname
  369. * The regular expression to extract the feature.uniquename from the fasta
  370. * header.
  371. * @param $re_accession
  372. * The regular expression to extract the accession of the feature.dbxref_id.
  373. * @param $db_id
  374. * The database ID of the above accession.
  375. * @param $rel_type
  376. * The type of relationship when creating a feature_relationship between
  377. * this feature (object) and an extracted subject.
  378. * @param $re_subject
  379. * The regular expression to extract the uniquename of the feature to be
  380. * the subject of the above specified relationship.
  381. * @param $parent_type
  382. * The type of the parent feature.
  383. * @param $method
  384. * The method of feature adding. (ie: 'Insert only', 'Update only',
  385. * 'Insert and update').
  386. * @param $analysis_id
  387. * The analysis_id to associate the features in this fasta file with.
  388. * @param $match_type
  389. * Whether to match existing features based on the 'Name' or 'Unique name'.
  390. */
  391. private function loadFasta($file_path, $organism_id, $type, $re_name, $re_uname, $re_accession,
  392. $db_id, $rel_type, $re_subject, $parent_type, $method, $analysis_id, $match_type) {
  393. // First get the type for this sequence.
  394. $cvtermsql = "
  395. SELECT CVT.cvterm_id
  396. FROM {cvterm} CVT
  397. INNER JOIN {cv} CV on CVT.cv_id = CV.cv_id
  398. LEFT JOIN {cvtermsynonym} CVTS on CVTS.cvterm_id = CVT.cvterm_id
  399. WHERE cv.name = :cvname and (CVT.name = :name or CVTS.synonym = :synonym)
  400. ";
  401. $cvterm = chado_query($cvtermsql, [
  402. ':cvname' => 'sequence',
  403. ':name' => $type,
  404. ':synonym' => $type,
  405. ])->fetchObject();
  406. if (!$cvterm) {
  407. $this->logMessage("Cannot find the term type: '!type'", ['!type' => $type], TRIPAL_ERROR);
  408. return 0;
  409. }
  410. // Second, if there is a parent type then get that.
  411. $parentcvterm = NULL;
  412. if ($parent_type) {
  413. $parentcvterm = chado_query($cvtermsql, [
  414. ':cvname' => 'sequence',
  415. ':name' => $parent_type,
  416. ':synonym' => $parent_type,
  417. ])->fetchObject();
  418. if (!$parentcvterm) {
  419. $this->logMessage("Cannot find the parent term type: '!type'",
  420. ['!type' => $parentcvterm], TRIPAL_ERROR);
  421. return 0;
  422. }
  423. }
  424. // Third, if there is a relationship type then get that.
  425. $relcvterm = NULL;
  426. if ($rel_type) {
  427. $relcvterm = chado_query($cvtermsql, [
  428. ':cvname' => 'sequence',
  429. ':name' => $rel_type,
  430. ':synonym' => $rel_type,
  431. ])->fetchObject();
  432. if (!$relcvterm) {
  433. $this->logMessage("Cannot find the relationship term type: '!type'",
  434. ['!type' => $relcvterm], TRIPAL_ERROR);
  435. return 0;
  436. }
  437. }
  438. // We need to get the table schema to make sure we don't overrun the
  439. // size of fields with what our regular expressions retrieve
  440. $feature_tbl = chado_get_schema('feature');
  441. $dbxref_tbl = chado_get_schema('dbxref');
  442. $this->logMessage(t("Step 1: Finding sequences..."));
  443. $filesize = filesize($file_path);
  444. $fh = fopen($file_path, 'r');
  445. if (!$fh) {
  446. throw new Exception(t("Cannot open file: !dfile", ['!dfile' => $file_path]));
  447. }
  448. $num_read = 0;
  449. // Iterate through the lines of the file. Keep a record for
  450. // where in the file each line is at for later import.
  451. $seqs = [];
  452. $num_seqs = 0;
  453. $prev_pos = 0;
  454. $set_start = FALSE;
  455. while ($line = fgets($fh)) {
  456. $num_read += strlen($line);
  457. // If we encounter a definition line then get the name, uniquename,
  458. // accession and relationship subject from the definition line.
  459. if (preg_match('/^>/', $line)) {
  460. // Remove the > symbol from the defline.
  461. $defline = preg_replace("/^>/", '', $line);
  462. // Get the feature name if a regular expression is provided.
  463. $name = "";
  464. if ($re_name) {
  465. if (!preg_match("/$re_name/", $defline, $matches)) {
  466. $this->logMessage("Regular expression for the feature name finds nothing. Line !line.",
  467. ['!line' => $i], TRIPAL_ERROR);
  468. }
  469. elseif (strlen($matches[1]) > $feature_tbl['fields']['name']['length']) {
  470. $this->logMessage("Regular expression retrieves a value too long for the feature name. Line !line.",
  471. ['!line' => $i], TRIPAL_WARNING);
  472. }
  473. else {
  474. $name = trim($matches[1]);
  475. }
  476. }
  477. // If the match_type is name and no regular expression was provided
  478. // then use the first word as the name, otherwise we don't set the name.
  479. elseif (strcmp($match_type, 'Name') == 0) {
  480. if (preg_match("/^\s*(.*?)[\s\|].*$/", $defline, $matches)) {
  481. if (strlen($matches[1]) > $feature_tbl['fields']['name']['length']) {
  482. $this->logMessage("Regular expression retrieves a feature name too long for the feature name. Line !line.",
  483. ['!line' => $i], TRIPAL_WARNING);
  484. }
  485. else {
  486. $name = trim($matches[1]);
  487. }
  488. }
  489. else {
  490. $this->logMessage("Cannot find a feature name. Line !line.", ['!line' => $i], TRIPAL_WARNING);
  491. }
  492. }
  493. // Get the feature uniquename if a regular expression is provided.
  494. $uname = "";
  495. if ($re_uname) {
  496. if (!preg_match("/$re_uname/", $defline, $matches)) {
  497. $this->logMessage("Regular expression for the feature unique name finds nothing. Line !line.",
  498. ['!line' => $i], TRIPAL_ERROR);
  499. }
  500. $uname = trim($matches[1]);
  501. }
  502. // If the match_type is name and no regular expression was provided
  503. // then use the first word as the name, otherwise, we don't set the
  504. // uniquename.
  505. elseif (strcmp($match_type, 'Unique name') == 0) {
  506. if (preg_match("/^\s*(.*?)[\s\|].*$/", $defline, $matches)) {
  507. $uname = trim($matches[1]);
  508. }
  509. else {
  510. $this->logMessage("Cannot find a feature unique name. Line !line.",
  511. ['!line' => $i], TRIPAL_ERROR);
  512. }
  513. }
  514. // Get the accession if a regular expression is provided.
  515. $accession = "";
  516. if (!empty($re_accession)) {
  517. preg_match("/$re_accession/", $defline, $matches);
  518. if (strlen($matches[1]) > $dbxref_tbl['fields']['accession']['length']) {
  519. tripal_report_error('trp-fasta', TRIPAL_WARNING, "WARNING: Regular expression retrieves an accession too long for the feature name. " .
  520. "Cannot add cross reference. Line %line.", [
  521. '%line' => $i,
  522. ]);
  523. }
  524. else {
  525. $accession = trim($matches[1]);
  526. }
  527. }
  528. // Get the relationship subject
  529. $subject = $uname ? $uname : "";
  530. if (!empty($re_subject)) {
  531. preg_match("/$re_subject/", $line, $matches);
  532. $subject = trim($matches[1]);
  533. }
  534. // Add the details to the sequence.
  535. $seqs[$num_seqs] = [
  536. 'name' => $name,
  537. 'uname' => $uname,
  538. 'accession' => $accession,
  539. 'subject' => $subject,
  540. 'seq_start' => ftell($fh),
  541. ];
  542. $set_start = TRUE;
  543. // If this isn't the first sequence, then we want to specify where
  544. // the previous sequence ended.
  545. if ($num_seqs > 0) {
  546. $seqs[$num_seqs - 1]['seq_end'] = $prev_pos;
  547. }
  548. $num_seqs++;
  549. }
  550. // Keep the current file position so we can use it to set the sequence
  551. // ending position
  552. $prev_pos = ftell($fh);
  553. }
  554. // Set the end position for the last sequence.
  555. $seqs[$num_seqs - 1]['seq_end'] = $num_read - strlen($line);
  556. // Now that we know where the sequences are in the file we need to add them.
  557. $this->logMessage("Step 2: Importing sequences...");
  558. $this->logMessage("Found !num_seqs sequence(s).", ['!num_seqs' => $num_seqs]);
  559. $this->setTotalItems($num_seqs);
  560. $this->setItemsHandled(0);
  561. for ($j = 0; $j < $num_seqs; $j++) {
  562. $seq = $seqs[$j];
  563. //$this->logMessage("Importing !seqname.", array('!seqname' => $seq['name']));
  564. $source = NULL;
  565. $this->loadFastaFeature($fh, $seq['name'], $seq['uname'], $db_id,
  566. $seq['accession'], $seq['subject'], $rel_type, $parent_type,
  567. $analysis_id, $organism_id, $cvterm, $source, $method, $re_name,
  568. $match_type, $parentcvterm, $relcvterm, $seq['seq_start'],
  569. $seq['seq_end']);
  570. $this->setItemsHandled($j);
  571. }
  572. fclose($fh);
  573. $this->setItemsHandled($num_seqs);
  574. }
  575. /**
  576. * A helper function for loadFasta() to load a single feature
  577. *
  578. * @ingroup fasta_loader
  579. */
  580. private function loadFastaFeature($fh, $name, $uname, $db_id, $accession, $parent,
  581. $rel_type, $parent_type, $analysis_id, $organism_id, $cvterm, $source, $method, $re_name,
  582. $match_type, $parentcvterm, $relcvterm, $seq_start, $seq_end) {
  583. // Check to see if this feature already exists if the match_type is 'Name'.
  584. if (strcmp($match_type, 'Name') == 0) {
  585. $values = [
  586. 'organism_id' => $organism_id,
  587. 'name' => $name,
  588. 'type_id' => $cvterm->cvterm_id,
  589. ];
  590. $results = chado_select_record('feature', [
  591. 'feature_id',
  592. ], $values);
  593. if (count($results) > 1) {
  594. $this->logMessage("Multiple features exist with the name '!name' of type '!type' for the organism. skipping",
  595. ['!name' => $name, '!type' => $type], TRIPAL_ERROR);
  596. return 0;
  597. }
  598. if (count($results) == 1) {
  599. $feature = $results[0];
  600. }
  601. }
  602. // Check if this feature already exists if the match_type is 'Unique Name'.
  603. if (strcmp($match_type, 'Unique name') == 0) {
  604. $values = [
  605. 'organism_id' => $organism_id,
  606. 'uniquename' => $uname,
  607. 'type_id' => $cvterm->cvterm_id,
  608. ];
  609. $results = chado_select_record('feature', ['feature_id'], $values);
  610. if (count($results) > 1) {
  611. $this->logMessage("Multiple features exist with the name '!name' of type '!type' for the organism. skipping",
  612. ['!name' => $name, '!type' => $type], TRIPAL_WARNING);
  613. return 0;
  614. }
  615. if (count($results) == 1) {
  616. $feature = $results[0];
  617. }
  618. // If the feature exists but this is an "insert only" then skip.
  619. if (isset($feature) and (strcmp($method, 'Insert only') == 0)) {
  620. $this->logMessage("Feature already exists '!name' ('!uname') while matching on !type. Skipping insert.",
  621. [
  622. '!name' => $name,
  623. '!uname' => $uname,
  624. '!type' => drupal_strtolower($match_type),
  625. ], TRIPAL_WARNING);
  626. return 0;
  627. }
  628. }
  629. // If we don't have a feature and we're doing an insert then do the insert.
  630. $inserted = 0;
  631. if (!isset($feature) and (strcmp($method, 'Insert only') == 0 or strcmp($method, 'Insert and update') == 0)) {
  632. // If we have a unique name but not a name then set them to be the same
  633. if (!$uname) {
  634. $uname = $name;
  635. }
  636. elseif (!$name) {
  637. $name = $uname;
  638. }
  639. // Insert the feature record.
  640. $values = [
  641. 'organism_id' => $organism_id,
  642. 'name' => $name,
  643. 'uniquename' => $uname,
  644. 'type_id' => $cvterm->cvterm_id,
  645. ];
  646. $success = chado_insert_record('feature', $values);
  647. if (!$success) {
  648. $this->logMessage("Failed to insert feature '!name (!uname)'", [
  649. '!name' => $name,
  650. '!uname' => $numane,
  651. ], TRIPAL_ERROR);
  652. return 0;
  653. }
  654. // now get the feature we just inserted
  655. $values = [
  656. 'organism_id' => $organism_id,
  657. 'uniquename' => $uname,
  658. 'type_id' => $cvterm->cvterm_id,
  659. ];
  660. $results = chado_select_record('feature', ['feature_id'], $values);
  661. if (count($results) == 1) {
  662. $inserted = 1;
  663. $feature = $results[0];
  664. }
  665. else {
  666. $this->logMessage("Failed to retreive newly inserted feature '!name (!uname)'", [
  667. '!name' => $name,
  668. '!uname' => $numane,
  669. ], TRIPAL_ERRORR);
  670. return 0;
  671. }
  672. // Add the residues for this feature
  673. $this->loadFastaResidues($fh, $feature->feature_id, $seq_start, $seq_end);
  674. }
  675. // if we don't have a feature and the user wants to do an update then fail
  676. if (!isset($feature) and (strcmp($method, 'Update only') == 0 or strcmp($method, 'Insert and update') == 0)) {
  677. $this->logMessage("Failed to find feature '!name' ('!uname') while matching on " . drupal_strtolower($match_type) . ".",
  678. ['!name' => $name, '!uname' => $uname], TRIPAL_ERROR);
  679. return 0;
  680. }
  681. // if we do have a feature and this is an update then proceed with the update
  682. if (isset($feature) and !$inserted and (strcmp($method, 'Update only') == 0 or strcmp($method, 'Insert and update') == 0)) {
  683. // if the user wants to match on the Name field
  684. if (strcmp($match_type, 'Name') == 0) {
  685. // if we're matching on the name but do not have a unique name then we
  686. // don't want to update the uniquename.
  687. $values = [];
  688. if ($uname) {
  689. // First check to make sure that by changing the unique name of this
  690. // feature that we won't conflict with another existing feature of
  691. // the same name
  692. $values = [
  693. 'organism_id' => $organism_id,
  694. 'uniquename' => $uname,
  695. 'type_id' => $cvterm->cvterm_id,
  696. ];
  697. $results = chado_select_record('feature', ['feature_id'], $values);
  698. if (count($results) > 0) {
  699. $this->logMessage("Cannot update the feature '!name' with a uniquename of '!uname' and type of '!type' as it " .
  700. "conflicts with an existing feature with the same uniquename and type.",
  701. [
  702. '!name' => $name,
  703. '!uname' => $uname,
  704. '!type' => $type,
  705. ], TRIPAL_ERROR);
  706. return 0;
  707. }
  708. // the changes to the uniquename don't conflict so proceed with the update
  709. $values = ['uniquename' => $uname];
  710. $match = [
  711. 'name' => $name,
  712. 'organism_id' => $organism_id,
  713. 'type_id' => $cvterm->cvterm_id,
  714. ];
  715. // perform the update
  716. $success = chado_update_record('feature', $match, $values);
  717. if (!$success) {
  718. $this->logMessage("Failed to update feature '!name' ('!name')",
  719. ['!name' => $name, '!uiname' => $uname], TRIPAL_ERROR);
  720. return 0;
  721. }
  722. }
  723. }
  724. // If the user wants to match on the unique name field.
  725. if (strcmp($match_type, 'Unique name') == 0) {
  726. // If we're matching on the uniquename and have a new name then
  727. // we want to update the name.
  728. $values = [];
  729. if ($name) {
  730. $values = ['name' => $name];
  731. $match = [
  732. 'uniquename' => $uname,
  733. 'organism_id' => $organism_id,
  734. 'type_id' => $cvterm->cvterm_id,
  735. ];
  736. $success = chado_update_record('feature', $match, $values);
  737. if (!$success) {
  738. $this->logMessage("Failed to update feature '!name' ('!name')",
  739. ['!name' => $name, '!uiname' => $uname], TRIPAL_ERROR);
  740. return 0;
  741. }
  742. }
  743. }
  744. }
  745. // Update the residues for this feature
  746. $this->loadFastaResidues($fh, $feature->feature_id, $seq_start, $seq_end);
  747. // add in the analysis link
  748. if ($analysis_id) {
  749. // if the association doens't alredy exist then add one
  750. $values = [
  751. 'analysis_id' => $analysis_id,
  752. 'feature_id' => $feature->feature_id,
  753. ];
  754. $results = chado_select_record('analysisfeature', ['analysisfeature_id'], $values);
  755. if (count($results) == 0) {
  756. $success = chado_insert_record('analysisfeature', $values);
  757. if (!$success) {
  758. $this->logMessage("Failed to associate analysis and feature '!name' ('!name')",
  759. ['!name' => $name, '!uname' => $uname], TRIPAL_ERROR);
  760. return 0;
  761. }
  762. }
  763. }
  764. // now add the database cross reference
  765. if ($db_id) {
  766. // check to see if this accession reference exists, if not add it
  767. $values = [
  768. 'db_id' => $db_id,
  769. 'accession' => $accession,
  770. ];
  771. $results = chado_select_record('dbxref', ['dbxref_id'], $values);
  772. // if the accession doesn't exist then add it
  773. if (count($results) == 0) {
  774. $results = chado_insert_record('dbxref', $values);
  775. if (!$results) {
  776. $this->logMessage("Failed to add database accession '!accession'",
  777. ['!accession' => $accession], TRIPAL_ERROR);
  778. return 0;
  779. }
  780. $results = chado_select_record('dbxref', ['dbxref_id'], $values);
  781. if (count($results) == 1) {
  782. $dbxref = $results[0];
  783. }
  784. else {
  785. $this->logMessage("Failed to retreive newly inserted dbxref '!name (!uname)'",
  786. ['!name' => $name, '!uname' => $numane], TRIPAL_ERROR);
  787. return 0;
  788. }
  789. }
  790. else {
  791. $dbxref = $results[0];
  792. }
  793. // check to see if the feature dbxref record exists if not, then add it
  794. $values = [
  795. 'feature_id' => $feature->feature_id,
  796. 'dbxref_id' => $dbxref->dbxref_id,
  797. ];
  798. $results = chado_select_record('feature_dbxref', ['feature_dbxref_id'], $values);
  799. if (count($results) == 0) {
  800. $success = chado_insert_record('feature_dbxref', $values);
  801. if (!$success) {
  802. $this->logMessage("Failed to add associate database accession '!accession' with feature",
  803. ['!accession' => $accession], TRIPAL_ERROR);
  804. return 0;
  805. }
  806. }
  807. }
  808. // now add in the relationship if one exists. If not, then add it
  809. if ($rel_type) {
  810. $values = [
  811. 'organism_id' => $organism_id,
  812. 'uniquename' => $parent,
  813. 'type_id' => $parentcvterm->cvterm_id,
  814. ];
  815. $results = chado_select_record('feature', ['feature_id'], $values);
  816. if (count($results) != 1) {
  817. $this->logMessage("Cannot find a unique feature for the parent '!parent' of type '!type' for the feature.",
  818. ['!parent' => $parent, '!type' => $parent_type], TRIPAL_ERROR);
  819. return 0;
  820. }
  821. $parent_feature = $results[0];
  822. // check to see if the relationship already exists if not then add it
  823. $values = [
  824. 'subject_id' => $feature->feature_id,
  825. 'object_id' => $parent_feature->feature_id,
  826. 'type_id' => $relcvterm->cvterm_id,
  827. ];
  828. $results = chado_select_record('feature_relationship', ['feature_relationship_id'], $values);
  829. if (count($results) == 0) {
  830. $success = chado_insert_record('feature_relationship', $values);
  831. if (!$success) {
  832. $this->logMessage("Failed to add associate database accession '!accession' with feature",
  833. ['!accession' => $accession], TRIPAL_ERROR);
  834. return 0;
  835. }
  836. }
  837. }
  838. }
  839. /**
  840. * Adds the residues column to the feature.
  841. *
  842. * This function seeks to the proper location in the file for the sequence
  843. * and reads in chunks of sequence and appends them to the feature.residues
  844. * column in the database.
  845. *
  846. * @param $fh
  847. * @param $feature_id
  848. * @param $seq_start
  849. * @param $seq_end
  850. */
  851. private function loadFastaResidues($fh, $feature_id, $seq_start, $seq_end) {
  852. // First position the file at the beginning of the sequence
  853. fseek($fh, $seq_start, SEEK_SET);
  854. $chunk_size = 100000000;
  855. $chunk = '';
  856. $seqlen = ($seq_end - $seq_start);
  857. $num_read = 0;
  858. $total_seq_size = 0;
  859. // First, make sure we don't have a null in the residues
  860. $sql = "UPDATE {feature} SET residues = '' WHERE feature_id = :feature_id";
  861. chado_query($sql, [':feature_id' => $feature_id]);
  862. // Read in the lines until we reach the end of the sequence. Once we
  863. // get a specific bytes read then append the sequence to the one in the
  864. // database.
  865. $partial_seq_size = 0;
  866. $chunk_intv_read = 0;
  867. while ($line = fgets($fh)) {
  868. $num_read += strlen($line) + 1;
  869. $chunk_intv_read += strlen($line) + 1;
  870. $partial_seq_size += strlen($line);
  871. $chunk .= trim($line);
  872. // If we've read in enough of the sequence then append it to the database.
  873. if ($chunk_intv_read >= $chunk_size) {
  874. $sql = "
  875. UPDATE {feature}
  876. SET residues = residues || :chunk
  877. WHERE feature_id = :feature_id
  878. ";
  879. $success = chado_query($sql, [
  880. ':feature_id' => $feature_id,
  881. ':chunk' => $chunk,
  882. ]);
  883. if (!$success) {
  884. return FALSE;
  885. }
  886. $total_seq_size += strlen($chunk);
  887. $chunk = '';
  888. $chunk_intv_read = 0;
  889. }
  890. // If we've reached the end of the sequence then break out of the loop
  891. if (ftell($fh) == $seq_end) {
  892. break;
  893. }
  894. }
  895. // write the last bit of sequence if it remains
  896. if (strlen($chunk) > 0) {
  897. $sql = "
  898. UPDATE {feature}
  899. SET residues = residues || :chunk
  900. WHERE feature_id = :feature_id
  901. ";
  902. $success = chado_query($sql, [
  903. ':feature_id' => $feature_id,
  904. ':chunk' => $chunk,
  905. ]);
  906. if (!$success) {
  907. return FALSE;
  908. }
  909. $total_seq_size += $partial_seq_size;
  910. $partial_seq_size = 0;
  911. $chunk = '';
  912. $chunk_intv_read = 0;
  913. }
  914. // Now update the seqlen and md5checksum fields
  915. $sql = "UPDATE {feature} SET seqlen = char_length(residues), md5checksum = md5(residues) WHERE feature_id = :feature_id";
  916. chado_query($sql, [':feature_id' => $feature_id]);
  917. }
  918. }