gff_loader.php 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990
  1. <?php
  2. /**
  3. * @defgroup gff3_loader GFF3 Feature Loader
  4. * @{
  5. * Provides gff3 loading functionality. Creates features based on their specification in a GFF3 file.
  6. * @}
  7. * @ingroup tripal_feature
  8. */
  9. // TODO: The rank column on the feature_relationship table needs to be used to
  10. // make sure the ordering of CDS (exons) is correct.
  11. // The entries in the GFF file are not in order so the order of the relationships
  12. // is not put in correctly.
  13. /**
  14. *
  15. *
  16. * @ingroup gff3_loader
  17. */
  18. function tripal_core_gff3_load_form (){
  19. $form['gff_file']= array(
  20. '#type' => 'textfield',
  21. '#title' => t('GFF3 File'),
  22. '#description' => t('Please enter the full system path for the GFF file, or a path within the Drupal
  23. installation (e.g. /sites/default/files/xyz.gff). The path must be accessible to the
  24. server on which this Drupal instance is running.'),
  25. '#required' => TRUE,
  26. '#weight' => 1
  27. );
  28. // get the list of organisms
  29. $sql = "SELECT * FROM {organism} ORDER BY genus, species";
  30. $previous_db = tripal_db_set_active('chado'); // use chado database
  31. $org_rset = db_query($sql);
  32. tripal_db_set_active($previous_db); // now use drupal database
  33. $organisms = array();
  34. $organisms[''] = '';
  35. while($organism = db_fetch_object($org_rset)){
  36. $organisms[$organism->organism_id] = "$organism->genus $organism->species ($organism->common_name)";
  37. }
  38. $form['organism_id'] = array (
  39. '#title' => t('Organism'),
  40. '#type' => t('select'),
  41. '#description' => t("Choose the organism to which these sequences are associated "),
  42. '#required' => TRUE,
  43. '#options' => $organisms,
  44. );
  45. $form['import_options'] = array(
  46. '#type' => 'fieldset',
  47. '#title' => t('Import Options'),
  48. '#weight'=> 6,
  49. '#collapsed' => TRUE
  50. );
  51. $form['import_options']['add_only']= array(
  52. '#type' => 'checkbox',
  53. '#title' => t('Import only new features'),
  54. '#required' => FALSE,
  55. '#description' => t('The job will skip features in the GFF file that already
  56. exist in the database and import only new features.'),
  57. '#weight' => 2
  58. );
  59. $form['import_options']['update']= array(
  60. '#type' => 'checkbox',
  61. '#title' => t('Import all and update'),
  62. '#required' => FALSE,
  63. '#default_value' => 'checked',
  64. '#description' => t('Existing features will be updated and new features will be added. Attributes
  65. for a feature that are not present in the GFF but which are present in the
  66. database will not be altered.'),
  67. '#weight' => 3
  68. );
  69. $form['import_options']['refresh']= array(
  70. '#type' => 'checkbox',
  71. '#title' => t('Import all and replace'),
  72. '#required' => FALSE,
  73. '#description' => t('Existing features will be updated and feature properties not
  74. present in the GFF file will be removed.'),
  75. '#weight' => 4
  76. );
  77. $form['import_options']['remove']= array(
  78. '#type' => 'checkbox',
  79. '#title' => t('Delete features'),
  80. '#required' => FALSE,
  81. '#description' => t('Features present in the GFF file that exist in the database
  82. will be removed rather than imported'),
  83. '#weight' => 5
  84. );
  85. $form['analysis'] = array(
  86. '#type' => 'fieldset',
  87. '#title' => t('Analysis Used to Derive Features'),
  88. '#weight'=> 6,
  89. '#collapsed' => TRUE
  90. );
  91. $form['analysis']['desc'] = array(
  92. '#type' => 'markup',
  93. '#value' => t("Why specify an analysis for a data load? All data comes
  94. from some place, even if downloaded from Genbank. By specifying
  95. analysis details for all data uploads, it allows an end user to reproduce the
  96. data set, but at least indicates the source of the data."),
  97. );
  98. // get the list of organisms
  99. $sql = "SELECT * FROM {analysis} ORDER BY name";
  100. $previous_db = tripal_db_set_active('chado'); // use chado database
  101. $org_rset = db_query($sql);
  102. tripal_db_set_active($previous_db); // now use drupal database
  103. $analyses = array();
  104. $analyses[''] = '';
  105. while($analysis = db_fetch_object($org_rset)){
  106. $analyses[$analysis->analysis_id] = "$analysis->name ($analysis->program $analysis->programversion, $analysis->sourcename)";
  107. }
  108. $form['analysis']['analysis_id'] = array (
  109. '#title' => t('Analysis'),
  110. '#type' => t('select'),
  111. '#description' => t("Choose the analysis to which these features are associated "),
  112. '#required' => TRUE,
  113. '#options' => $analyses,
  114. );
  115. $form['button'] = array(
  116. '#type' => 'submit',
  117. '#value' => t('Import GFF3 file'),
  118. '#weight' => 10,
  119. );
  120. return $form;
  121. }
  122. /**
  123. *
  124. *
  125. * @ingroup gff3_loader
  126. */
  127. function tripal_core_gff3_load_form_validate ($form, &$form_state){
  128. $gff_file = $form_state['values']['gff_file'];
  129. $organism_id = $form_state['values']['organism_id'];
  130. $add_only = $form_state['values']['add_only'];
  131. $update = $form_state['values']['update'];
  132. $refresh = $form_state['values']['refresh'];
  133. $remove = $form_state['values']['remove'];
  134. // check to see if the file is located local to Drupal
  135. $dfile = $_SERVER['DOCUMENT_ROOT'] . base_path() . $gff_file;
  136. if(!file_exists($dfile)){
  137. // if not local to Drupal, the file must be someplace else, just use
  138. // the full path provided
  139. $dfile = $gff_file;
  140. }
  141. if(!file_exists($dfile)){
  142. form_set_error('gff_file',t("Cannot find the file on the system. Check that the file exists or that the web server has permissions to read the file."));
  143. }
  144. if (($add_only and ($update or $refresh or $remove)) or
  145. ($update and ($add_only or $refresh or $remove)) or
  146. ($refresh and ($update or $add_only or $remove)) or
  147. ($remove and ($update or $refresh or $add_only))){
  148. form_set_error('add_only',t("Please select only one checkbox from the import options section"));
  149. }
  150. }
  151. /**
  152. *
  153. * @ingroup gff3_loader
  154. */
  155. function tripal_core_gff3_load_form_submit ($form, &$form_state){
  156. global $user;
  157. $gff_file = $form_state['values']['gff_file'];
  158. $organism_id = $form_state['values']['organism_id'];
  159. $add_only = $form_state['values']['add_only'];
  160. $update = $form_state['values']['update'];
  161. $refresh = $form_state['values']['refresh'];
  162. $remove = $form_state['values']['remove'];
  163. $analysis_id = $form_state['values']['analysis_id'];
  164. $args = array($gff_file,$organism_id,$analysis_id,$add_only,$update,$refresh,$remove);
  165. $type = '';
  166. if($add_only){
  167. $type = 'import only new features';
  168. }
  169. if($update){
  170. $type = 'import all and update';
  171. }
  172. if($refresh){
  173. $type = 'import all and replace';
  174. }
  175. if($remove){
  176. $type = 'delete features';
  177. }
  178. tripal_add_job("$type GFF3 file $gff_file",'tripal_core',
  179. 'tripal_core_load_gff3',$args,$user->uid);
  180. return '';
  181. }
  182. /**
  183. *
  184. *
  185. * @ingroup gff3_loader
  186. */
  187. function tripal_core_load_gff3($gff_file, $organism_id,$analysis_id,$add_only =0,
  188. $update = 0, $refresh = 0, $remove = 0, $job = NULL)
  189. {
  190. // this array is used to cache all of the features in the GFF file and
  191. // used to lookup parent and target relationships
  192. $gff_features = array();
  193. // check to see if the file is located local to Drupal
  194. $dfile = $_SERVER['DOCUMENT_ROOT'] . base_path() . $gff_file;
  195. if(!file_exists($dfile)){
  196. // if not local to Drupal, the file must be someplace else, just use
  197. // the full path provided
  198. $dfile = $gff_file;
  199. }
  200. if(!file_exists($dfile)){
  201. print "ERROR: cannot find the file: $dfile\n";
  202. return 0;
  203. }
  204. $previous_db = tripal_db_set_active('chado');
  205. print "Opening $gff_file\n";
  206. $lines = file($dfile,FILE_SKIP_EMPTY_LINES);
  207. $i = 0;
  208. // get the controlled vocaubulary that we'll be using. The
  209. // default is the 'sequence' ontology
  210. $sql = "SELECT * FROM cv WHERE name = '%s'";
  211. $cv = db_fetch_object(db_query($sql,'sequence'));
  212. if(!$cv){
  213. print "ERROR: cannot find the 'sequence' ontology\n";
  214. return '';
  215. }
  216. // get the organism for which this GFF3 file belongs
  217. $sql = "SELECT * FROM organism WHERE organism_id = %d";
  218. $organism = db_fetch_object(db_query($sql,$organism_id));
  219. $num_lines = sizeof($lines);
  220. $interval = intval($num_lines * 0.01);
  221. if($interval == 0){
  222. $interval = $num_lines;
  223. }
  224. $in_fasta = 0;
  225. foreach ($lines as $line_num => $line) {
  226. $i++; // update the line count
  227. // update the job status every 1% features
  228. if($job and $i % $interval == 0){
  229. tripal_job_set_progress($job,intval(($i/$num_lines)*100));
  230. }
  231. // check to see if we have FASTA section, if so then set the variable
  232. // to start parsing
  233. if(preg_match('/^##FASTA/i',$line)){
  234. $in_fasta = 1;
  235. }
  236. // skip comments
  237. if(preg_match('/^#/',$line)){
  238. continue;
  239. }
  240. // skip empty lines
  241. if(preg_match('/^\s*$/',$line)){
  242. continue;
  243. }
  244. // handle FASTA section
  245. // TODO: remove URL encoding
  246. $cols = explode("\t",$line);
  247. if(sizeof($cols) != 9){
  248. print "ERROR: improper number of columns on line $i\n";
  249. print_r($cols);
  250. return '';
  251. }
  252. // get the column values
  253. $landmark = $cols[0];
  254. $source = $cols[1];
  255. $type = $cols[2];
  256. $start = $cols[3];
  257. $end = $cols[4];
  258. $score = $cols[5];
  259. $strand = $cols[6];
  260. $phase = $cols[7];
  261. $attrs = explode(";",$cols[8]); // split by a semi-colon
  262. // ready the start and stop for chado. Chado expects these positions
  263. // to be zero-based, so we substract 1 from the fmin
  264. $fmin = $start - 1;
  265. $fmax = $end;
  266. if($end < $start){
  267. $fmin = $end - 1;
  268. $fmax = $start;
  269. }
  270. // format the strand for chado
  271. if(strcmp($strand,'.')==0){
  272. $strand = 0;
  273. }
  274. elseif(strcmp($strand,'+')==0){
  275. $strand = 1;
  276. }
  277. elseif(strcmp($strand,'-')==0){
  278. $strand = -1;
  279. }
  280. if(strcmp($phase,'.')==0){
  281. $phase = '';
  282. }
  283. // get the type record
  284. $cvtermsql = "SELECT CVT.cvterm_id, CVT.cv_id, CVT.name, CVT.definition,
  285. CVT.dbxref_id, CVT.is_obsolete, CVT.is_relationshiptype
  286. FROM {cvterm} CVT
  287. INNER JOIN {cv} CV on CVT.cv_id = CV.cv_id
  288. LEFT JOIN {cvtermsynonym} CVTS on CVTS.cvterm_id = CVT.cvterm_id
  289. WHERE CV.cv_id = %d and (CVT.name = '%s' or CVTS.synonym = '%s')";
  290. $cvterm = db_fetch_object(db_query($cvtermsql,$cv->cv_id,$type,$type));
  291. if(!$cvterm){
  292. print "ERROR: cannot find ontology term '$type' on line $i.\n";
  293. return '';
  294. }
  295. // break apart each of the attributes
  296. $tags = array();
  297. $attr_name = '';
  298. $attr_uniquename = '';
  299. $attr_residue_info = '';
  300. $attr_locgroup = 0;
  301. $attr_fmin_partial = 'f';
  302. $attr_fmax_partial = 'f';
  303. $attr_is_obsolete = 'f';
  304. $attr_is_analysis = 'f';
  305. $attr_others = '';
  306. $residues = '';
  307. foreach($attrs as $attr){
  308. $attr = rtrim($attr);
  309. $attr = ltrim($attr);
  310. if(strcmp($attr,'')==0){
  311. continue;
  312. }
  313. if(!preg_match('/^[^\=]+\=[^\=]+$/',$attr)){
  314. print "ERROR: attribute is not correctly formatted on line $i: $attr\n";
  315. return '';
  316. }
  317. // break apart each tag
  318. $tag = explode("=",$attr); // split by equals sign
  319. // multiple instances of an attribute are separated by commas
  320. $tags[$tag[0]] = explode(",",$tag[1]); // split by comma
  321. if(strcmp($tag[0],'ID')==0){
  322. $attr_uniquename = $tag[1];
  323. }
  324. elseif(strcmp($tag[0],'Name')==0){
  325. $attr_name = $tag[1];
  326. }
  327. // get the list of other attributes other than those reserved ones.
  328. elseif(strcmp($tag[0],'Alias')!=0 and strcmp($tag[0],'Parent')!=0 and
  329. strcmp($tag[0],'Target')!=0 and strcmp($tag[0],'Gap')!=0 and
  330. strcmp($tag[0],'Derives_from')!=0 and strcmp($tag[0],'Note')!=0 and
  331. strcmp($tag[0],'Dbxref')!=0 and strcmp($tag[0],'Ontology_term')!=0 and
  332. strcmp($tag[0],'Is_circular')!=0){
  333. $attr_others[$tag[0]] = $tag[1];
  334. }
  335. }
  336. // if neither name nor uniquename are provided then generate one
  337. if(!$attr_uniquename and !$attr_name){
  338. if(array_key_exists('Parent',$tags)){
  339. $attr_uniquename = $tags['Parent'][0]."-$type-$landmark:$fmin..$fmax";
  340. } else {
  341. print "ERROR: cannot generate a uniquename for feature on line $i\n";
  342. exit;
  343. }
  344. $attr_name = $attr_uniquename;
  345. }
  346. // if a name is not specified then use the unique name
  347. if(strcmp($attr_name,'')==0){
  348. $attr_name = $attr_uniquename;
  349. }
  350. // if an ID attribute is not specified then use the attribute name and
  351. // hope for the best
  352. if(!$attr_uniquename){
  353. $attr_uniquename = $attr_name;
  354. }
  355. // make sure the landmark sequence exists in the database. We don't
  356. // know the type of the landmark so we'll hope that it's unique across
  357. // all types. If not we'll error out. This test is only necessary if
  358. // if the landmark and the uniquename are different. If they are the same
  359. // then this is the information for the landmark
  360. if(strcmp($landmark,$attr_uniquename)!=0){
  361. $feature_sql = "SELECT count(*) as num_landmarks
  362. FROM {feature}
  363. WHERE organism_id = %d and uniquename = '%s'";
  364. $count = db_fetch_object(db_query($feature_sql,$organism_id,$landmark));
  365. if(!$count or $count->num_landmarks == 0){
  366. print "ERROR: the landmark '$landmark' cannot be found for this organism. ".
  367. "Please add the landmark and then retry the import of this GFF3 ".
  368. "file.\n";
  369. return '';
  370. }
  371. if($count->num_landmarks > 1){
  372. print "ERROR: the landmark '$landmark' is not unique for this organism. ".
  373. "The features cannot be associated.\n";
  374. return '';
  375. }
  376. }
  377. // if the option is to remove or refresh then we want to remove
  378. // the feature from the database.
  379. if($remove or $refresh){
  380. print "Removing feature '$attr_uniquename'\n";
  381. $sql = "DELETE FROM {feature}
  382. WHERE organism_id = %d and uniquename = '%s' and type_id = %d";
  383. $result = db_query($sql,$organism->organism_id,$attr_uniquename,$cvterm->cvterm_id);
  384. if(!$result){
  385. print "ERROR: cannot delete feature $attr_uniquename\n";
  386. }
  387. $feature = 0;
  388. }
  389. // add or update the feature and all properties
  390. if($update or $refresh or $add_only){
  391. // add/update the feature
  392. print "$i ";
  393. $feature = tripal_core_load_gff3_feature($organism,$analysis_id,$cvterm,
  394. $attr_uniquename,$attr_name,$residues,$attr_is_analysis,
  395. $attr_is_obsolete, $add_only);
  396. // store all of the features for use later by parent and target
  397. // relationships
  398. $gff_features[$feature->uniquename]['type'] = $type;
  399. $gff_features[$feature->uniquename]['strand'] = $strand;
  400. if($feature){
  401. // add/update the featureloc if the landmark and the ID are not the same
  402. // if they are the same then this entry in the GFF is probably a landmark identifier
  403. if(strcmp($landmark,$attr_uniquename)!=0){
  404. tripal_core_load_gff3_featureloc($feature,$organism,
  405. $landmark,$fmin,$fmax,$strand,$phase,$attr_fmin_partial,
  406. $attr_fmax_partial,$attr_residue_info,$attr_locgroup);
  407. }
  408. // add any aliases for this feature
  409. if(array_key_exists('Alias',$tags)){
  410. tripal_core_load_gff3_alias($feature,$tags['Alias']);
  411. }
  412. // add any aliases for this feature
  413. if(array_key_exists('Dbxref',$tags)){
  414. tripal_core_load_gff3_dbxref($feature,$tags['Dbxref']);
  415. }
  416. // add any aliases for this feature
  417. if(array_key_exists('Ontology_term',$tags)){
  418. tripal_core_load_gff3_ontology($feature,$tags['Ontology_term']);
  419. }
  420. // add parent relationships
  421. if(array_key_exists('Parent',$tags)){
  422. tripal_core_load_gff3_parents($feature,$cvterm,$tags['Parent'],$gff_features,$organism_id,$fmin);
  423. }
  424. // add in the GFF3_source dbxref so that GBrowse can find the feature using the source column
  425. $source_ref = array('GFF_source:'.$source);
  426. tripal_core_load_gff3_dbxref($feature,$source_ref);
  427. }
  428. }
  429. }
  430. // now set the rank of any parent/child relationships. The order is based
  431. // on the fmin. The start rank is 1. This allows features with other
  432. // relationships to be '0' (the default), and doesn't interfer with the
  433. // ordering defined here.
  434. foreach($gff_features as $parent => $details){
  435. // only iterate through parents that have children
  436. if($details['children']){
  437. // get the parent
  438. $values = array(
  439. 'uniquename' => $parent,
  440. 'type_id' => array(
  441. 'cv_id' => array(
  442. 'name' => 'sequence'
  443. ),
  444. 'name' => $details['type'],
  445. ),
  446. 'organism_id' => $organism->organism_id,
  447. );
  448. $pfeature = tripal_core_chado_select('feature',array('*'),$values);
  449. // sort the children by order of their fmin positions (values of assoc. array)
  450. // if the parent is on the reverse strand then sort in reverse
  451. if($details['strand'] == -1){
  452. arsort($details['children']);
  453. } else {
  454. asort($details['children']);
  455. }
  456. // now iterate through the children and set their rank
  457. $rank = 1;
  458. print "Updating child ranks for $parent (".$details['type'].")\n";
  459. foreach($details['children'] as $kfeature_id => $kfmin){
  460. $match = array(
  461. 'object_id' => $pfeature[0]->feature_id,
  462. 'subject_id' => $kfeature_id,
  463. 'type_id' => array(
  464. 'cv_id' => array(
  465. 'name' => 'relationship'
  466. ),
  467. 'name' => 'part_of',
  468. ),
  469. );
  470. $values = array(
  471. 'rank' => $rank,
  472. );
  473. tripal_core_chado_update('feature_relationship',$match,$values);
  474. $rank++;
  475. }
  476. }
  477. }
  478. tripal_db_set_active($previous_db);
  479. return '';
  480. }
  481. /**
  482. *
  483. *
  484. * @ingroup gff3_loader
  485. */
  486. function tripal_core_load_gff3_parents($feature,$cvterm,$parents,&$gff_features,$organism_id,$fmin){
  487. $uname = $feature->uniquename;
  488. $type = $cvterm->name;
  489. $rel_type = 'part_of';
  490. // create these SQL statements that will be used repeatedly below.
  491. $cvtermsql = "SELECT CVT.cvterm_id
  492. FROM {cvterm} CVT
  493. INNER JOIN {cv} CV on CVT.cv_id = CV.cv_id
  494. LEFT JOIN {cvtermsynonym} CVTS on CVTS.cvterm_id = CVT.cvterm_id
  495. WHERE cv.name = '%s' and (CVT.name = '%s' or CVTS.synonym = '%s')";
  496. $feature_sql = "SELECT * FROM {feature}
  497. WHERE organism_id = %d and uniquename = '%s' and type_id = %d";
  498. // iterate through the parents in the list
  499. foreach($parents as $parent){
  500. $parent_type = $gff_features[$parent]['type'];
  501. // try to find the parent
  502. $parentcvterm = db_fetch_object(db_query($cvtermsql,'sequence',$parent_type,$parent_type));
  503. $relcvterm = db_fetch_object(db_query($cvtermsql,'relationship',$rel_type,$rel_type));
  504. $parent_feature = db_fetch_object(db_query($feature_sql,$organism_id,$parent,$parentcvterm->cvterm_id));
  505. // we want to add this feature to the child list for the parent
  506. // when the loader finishes, it will go back through the parent
  507. // features and rank the children by position
  508. $gff_features[$parent]['children'][$feature->feature_id] = $fmin;
  509. // if the parent exists then add the relationship otherwise print error and skip
  510. if($parent_feature){
  511. // check to see if the relationship already exists
  512. $sql = "SELECT * FROM {feature_relationship} WHERE subject_id = %d and object_id = %d and type_id = %d";
  513. $rel = db_fetch_object(db_query($sql,$feature->feature_id,$parent_feature->feature_id,$relcvterm->cvterm_id));
  514. if($rel){
  515. print " Relationship already exists, skipping '$uname' ($type) $rel_type '$parent' ($parent_type)\n";
  516. } else {
  517. // the relationship doesn't already exist, so add it.
  518. $sql = "INSERT INTO {feature_relationship} (subject_id,object_id,type_id)
  519. VALUES (%d,%d,%d)";
  520. $result = db_query($sql,$feature->feature_id,$parent_feature->feature_id,$relcvterm->cvterm_id);
  521. if(!$result){
  522. print "WARNING: failed to insert feature relationship '$uname' ($type) $rel_type '$parent' ($parent_type)\n";
  523. } else {
  524. print " Inserted relationship relationship: '$uname' ($type) $rel_type '$parent' ($parent_type)\n";
  525. }
  526. }
  527. }
  528. else {
  529. print "WARNING: cannot establish relationship '$uname' ($type) $rel_type '$parent' ($parent_type): Cannot find the parent\n";
  530. }
  531. }
  532. }
  533. /**
  534. *
  535. *
  536. * @ingroup gff3_loader
  537. */
  538. function tripal_core_load_gff3_dbxref($feature,$dbxrefs){
  539. // iterate through each of the dbxrefs
  540. foreach($dbxrefs as $dbxref){
  541. // get the database name from the reference. If it doesn't exist then create one.
  542. $ref = explode(":",$dbxref);
  543. $dbname = $ref[0];
  544. $accession = $ref[1];
  545. // first look for the database name if it doesn't exist then create one.
  546. // first check for the fully qualified URI (e.g. DB:<dbname>. If that
  547. // can't be found then look for the name as is. If it still can't be found
  548. // the create the database
  549. $db = tripal_core_chado_select('db',array('db_id'),array('name' => "DB:$dbname"));
  550. if(sizeof($db) == 0){
  551. $db = tripal_core_chado_select('db',array('db_id'),array('name' => "$dbname"));
  552. }
  553. if(sizeof($db) == 0){
  554. $ret = tripal_core_chado_insert('db',array('name' => $dbname,
  555. 'description' => 'Added automatically by the GFF loader'));
  556. if($ret){
  557. print " Added new database: $dbname\n";
  558. $db = tripal_core_chado_select('db',array('db_id'),array('name' => "$dbname"));
  559. } else {
  560. print "ERROR: cannot find or add the database $dbname\n";
  561. return 0;
  562. }
  563. }
  564. $db = $db[0];
  565. // now check to see if the accession exists
  566. $dbxref = tripal_core_chado_select('dbxref',array('dbxref_id'),array(
  567. 'accession' => $accession,'db_id' => $db->db_id));
  568. // if the accession doesn't exist then we want to add it
  569. if(sizeof($dbxref) == 0){
  570. $ret = tripal_core_chado_insert('dbxref',array('db_id' => $db->db_id,
  571. 'accession' => $accession,'version' => ''));
  572. $dbxref = tripal_core_chado_select('dbxref',array('dbxref_id'),array(
  573. 'accession' => $accession,'db_id' => $db->db_id));
  574. }
  575. $dbxref = $dbxref[0];
  576. // check to see if this feature dbxref already exists
  577. $fdbx = tripal_core_chado_select('feature_dbxref',array('feature_dbxref_id'),
  578. array('dbxref_id' => $dbxref->dbxref_id,'feature_id' => $feature->feature_id));
  579. // now associate this feature with the database reference if it doesn't
  580. // already exist
  581. if(sizeof($fdbx)==0){
  582. $ret = tripal_core_chado_insert('feature_dbxref',array(
  583. 'feature_id' => $feature->feature_id,
  584. 'dbxref_id' => $dbxref->dbxref_id));
  585. if($ret){
  586. print " Adding Dbxref $dbname:$accession\n";
  587. } else {
  588. print "ERROR: failed to insert Dbxref: $dbname:$accession\n";
  589. return 0;
  590. }
  591. } else {
  592. print " Dbxref already associated, skipping $dbname:$accession\n";
  593. }
  594. }
  595. return 1;
  596. }
  597. /**
  598. *
  599. *
  600. * @ingroup gff3_loader
  601. */
  602. function tripal_core_load_gff3_ontology($feature,$dbxrefs){
  603. // iterate through each of the dbxrefs
  604. foreach($dbxrefs as $dbxref){
  605. // get the database name from the reference. If it doesn't exist then create one.
  606. $ref = explode(":",$dbxref);
  607. $dbname = $ref[0];
  608. $accession = $ref[1];
  609. // first look for the database name
  610. $db = tripal_core_chado_select('db',array('db_id'),array('name' => "DB:$dbname"));
  611. if(sizeof($db) == 0){
  612. $db = tripal_core_chado_select('db',array('db_id'),array('name' => "$dbname"));
  613. }
  614. if(sizeof($db) == 0){
  615. print "ERROR: Database, $dbname is missing for reference: $dbname:$accession\n";
  616. return 0;
  617. }
  618. $db = $db[0];
  619. // now check to see if the accession exists
  620. $dbxref = tripal_core_chado_select('dbxref',array('dbxref_id'),array(
  621. 'accession' => $accession,'db_id' => $db->db_id));
  622. if(sizeof($dbxref) == 0){
  623. print "ERROR: Accession, $accession is missing for reference: $dbname:$accession\n";
  624. return 0;
  625. }
  626. $dbxref = $dbxref[0];
  627. // now check to see if the cvterm exists
  628. $cvterm = tripal_core_chado_select('cvterm',array('cvterm_id'),array(
  629. 'dbxref_id' => $dbxref->dbxref_id));
  630. // if it doesn't exist in the cvterm table, look for an alternate id
  631. if(sizeof($cvterm) == 0){
  632. $cvterm = tripal_core_chado_select('cvterm_dbxref',array('cvterm_id'),array(
  633. 'dbxref_id' => $dbxref->dbxref_id));
  634. }
  635. if(sizeof($cvterm) == 0){
  636. print "ERROR: CVTerm is missing for reference: $dbname:$accession\n";
  637. return 0;
  638. }
  639. $cvterm = $cvterm[0];
  640. // check to see if this feature cvterm already exists
  641. $fcvt = tripal_core_chado_select('feature_cvterm',array('feature_cvterm_id'),
  642. array('cvterm_id' => $cvterm->cvterm_id,'feature_id' => $feature->feature_id));
  643. // now associate this feature with the cvterm if it doesn't already exist
  644. if(sizeof($fcvt)==0){
  645. $values = array(
  646. 'feature_id' => $feature->feature_id,
  647. 'cvterm_id' => $cvterm->cvterm_id,
  648. 'pub_id' => array(
  649. 'uniquename' => 'null',
  650. ),
  651. );
  652. $ret = tripal_core_chado_insert('feature_cvterm',$values);
  653. if($ret){
  654. print " Adding ontology term $dbname:$accession\n";
  655. } else {
  656. print "ERROR: failed to insert ontology term: $dbname:$accession\n";
  657. return 0;
  658. }
  659. } else {
  660. print " Ontology term already associated, skipping $dbname:$accession\n";
  661. }
  662. }
  663. return 1;
  664. }
  665. /**
  666. *
  667. *
  668. * @ingroup gff3_loader
  669. */
  670. function tripal_core_load_gff3_alias($feature,$aliases){
  671. // make sure we have a 'synonym_type' vocabulary
  672. $sql = "SELECT * FROM {cv} WHERE name='synonym_type'";
  673. $syncv = db_fetch_object(db_query($sql));
  674. if(!$syncv){
  675. $sql = "INSERT INTO {cv} (name,definition) VALUES ('synonym_type','')";
  676. if(!db_query($sql)){
  677. print("ERROR: Failed to add the synonyms type vocabulary");
  678. return 0;
  679. }
  680. $syncv = db_fetch_object(db_query($sql));
  681. }
  682. // get the 'exact' cvterm, which is the type of synonym we're adding
  683. $cvtsql = "
  684. SELECT * FROM {cvterm} CVT
  685. INNER JOIN {cv} CV ON CVT.cv_id = CV.cv_id
  686. WHERE CVT.name = '%s' and CV.name = '%s'
  687. ";
  688. $syntype = db_fetch_object(db_query($cvtsql,'exact','synonym_type'));
  689. if(!$syntype){
  690. $term = array(
  691. 'name' => array('exact'),
  692. 'id' => array("internal:exact"),
  693. 'definition' => array(''),
  694. 'is_obsolete' => array(0),
  695. );
  696. $syntype = tripal_cv_obo_add_cv_term($term,$syncv,0,1);
  697. if(!$syntype){
  698. tripal_cv_obo_quiterror("Cannot add synonym type: internal:$type");
  699. }
  700. }
  701. // iterate through all of the aliases and add each one
  702. foreach($aliases as $alias){
  703. print " Adding Alias $alias\n";
  704. // check to see if the alias already exists in the synonym table
  705. // if not, then add it
  706. $synsql = "SELECT * FROM {synonym}
  707. WHERE name = '%s' and type_id = %d";
  708. $synonym = db_fetch_object(db_query($synsql,$alias,$syntype->cvterm_id));
  709. if(!$synonym){
  710. $sql = "INSERT INTO {synonym}
  711. (name,type_id,synonym_sgml)
  712. VALUES ('%s',%d,'%s')";
  713. $result = db_query($sql,$alias,$syntype->cvterm_id,'');
  714. if(!$result){
  715. print "ERROR: cannot add alias $alias to synonym table\n";
  716. }
  717. }
  718. $synonym = db_fetch_object(db_query($synsql,$alias,$syntype->cvterm_id));
  719. // check to see if we have a NULL publication in the pub table. If not,
  720. // then add one.
  721. $pubsql = "SELECT * FROM {pub} WHERE uniquename = 'null'";
  722. $pub = db_fetch_object(db_query($pubsql));
  723. if(!$pub){
  724. $sql = "INSERT INTO pub (uniquename,type_id) VALUES ('%s',
  725. (SELECT cvterm_id
  726. FROM cvterm CVT
  727. INNER JOIN dbxref DBX on DBX.dbxref_id = CVT.dbxref_id
  728. INNER JOIN db DB on DB.db_id = DBX.db_id
  729. WHERE CVT.name = 'null' and DB.name = 'null')";
  730. $result = db_query($sql,'null');
  731. if(!$result){
  732. print "ERROR: cannot add null publication needed for setup of alias\n";
  733. return 0;
  734. }
  735. }
  736. $pub = db_fetch_object(db_query($pubsql));
  737. // check to see if the synonym exists in the feature_synonym table
  738. // if not, then add it.
  739. $synsql = "SELECT * FROM {feature_synonym}
  740. WHERE synonym_id = %d and feature_id = %d and pub_id = %d";
  741. $fsyn = db_fetch_object(db_query($synsql,$synonym->synonym_id,$feature->feature_id,$pub->pub_id));
  742. if(!$fsyn){
  743. $sql = "INSERT INTO {feature_synonym}
  744. (synonym_id,feature_id,pub_id)
  745. VALUES (%d,%d,%d)";
  746. $result = db_query($sql,$synonym->synonym_id,$feature->feature_id,$pub->pub_id);
  747. if(!$result){
  748. print "ERROR: cannot add alias $alias to feature synonym table\n";
  749. return 0;
  750. }
  751. } else {
  752. print "Synonym $alias already exists. Skipping\n";
  753. }
  754. }
  755. return 1;
  756. }
  757. /**
  758. *
  759. *
  760. * @ingroup gff3_loader
  761. */
  762. function tripal_core_load_gff3_feature($organism,$analysis_id,$cvterm,$uniquename,$name,
  763. $residues,$is_analysis='f',$is_obsolete='f',$add_only) {
  764. // check to see if the feature already exists
  765. $feature_sql = "SELECT * FROM {feature}
  766. WHERE organism_id = %d and uniquename = '%s' and type_id = %d";
  767. $feature = db_fetch_object(db_query($feature_sql,$organism->organism_id,$uniquename,$cvterm->cvterm_id));
  768. if(strcmp($is_obsolete,'f')==0){
  769. $is_obsolete = 'false';
  770. }
  771. if(strcmp($is_analysis,'f')==0){
  772. $is_analysis = 'false';
  773. }
  774. // insert the feature if it does not exist otherwise perform an update
  775. if(!$feature){
  776. print "Adding feature '$uniquename' ($cvterm->name)\n";
  777. $isql = "INSERT INTO {feature} (organism_id, name, uniquename, residues, seqlen,
  778. md5checksum, type_id,is_analysis,is_obsolete)
  779. VALUES(%d,'%s','%s','%s',%d, '%s', %d, %s, %s)";
  780. $result = db_query($isql,$organism->organism_id,$name,$uniquename,$residues,strlen($residues),
  781. md5($residues),$cvterm->cvterm_id,$is_analysis,$is_obsolete);
  782. if(!$result){
  783. print "ERROR: failed to insert feature '$uniquename' ($cvterm->name)\n";
  784. return 0;
  785. }
  786. }
  787. elseif(!$add_only) {
  788. print "Updating feature '$uniquename' ($cvterm->name)\n";
  789. $usql = "UPDATE {feature}
  790. SET name = '%s', residues = '%s', seqlen = '%s', md5checksum = '%s',
  791. is_analysis = %s, is_obsolete = %s
  792. WHERE organism_id = %d and uniquename = '%s' and type_id = %d";
  793. $result = db_query($usql,$name,$residues,strlen($residues),md5($residues),$is_analysis,$is_obsolete,
  794. $organism_id,$uniquename,$cvterm->cvterm_id);
  795. if(!$result){
  796. print "ERROR: failed to update feature '$uniquename' ($cvterm->name)\n";
  797. return 0;
  798. }
  799. }
  800. else {
  801. // the feature exists and we don't want to update it so return
  802. // a value of 0. This will stop all downstream property additions
  803. print "Skipping existing feature: '$uniquename' ($cvterm->name).\n";
  804. return 0;
  805. }
  806. // get the newly added feature
  807. $feature = db_fetch_object(db_query($feature_sql,$organism->organism_id,$uniquename,$cvterm->cvterm_id));
  808. // add the analysisfeature entry to the analysisfeature table if it doesn't already exist
  809. $af_values = array('analysis_id' => $analysis_id, 'feature_id' => $feature->feature_id);
  810. if(tripal_core_chado_select('analysisfeature',array('analysisfeature_id'),$af_values,array('has_record'))){
  811. if(!tripal_core_chado_insert('analysisfeature',$af_values)){
  812. print "ERROR: could not add analysisfeature record: $analysis_id, $feature->feature_id\n";
  813. } else {
  814. print " Added analysisfeature record\n";
  815. }
  816. }
  817. return $feature;
  818. }
  819. /**
  820. *
  821. *
  822. * @ingroup gff3_loader
  823. */
  824. function tripal_core_load_gff3_featureloc($feature,$organism,$landmark,$fmin,
  825. $fmax,$strand,$phase,$is_fmin_partial,$is_fmax_partial,$residue_info,$locgroup)
  826. {
  827. // get the source feature
  828. $sql = "SELECT * FROM {feature}
  829. WHERE organism_id = %d and uniquename = '%s'";
  830. $srcfeature = db_fetch_object(db_query($sql,$organism->organism_id,$landmark));
  831. if(!$srcfeature){
  832. print "ERROR: cannot find landmark feature $landmark. Cannot add the feature location record\n";
  833. return 0;
  834. }
  835. // TODO: create an attribute that recognizes the residue_info,locgroup, is_fmin_partial and is_fmax_partial, right now these are
  836. // hardcoded to be false and 0 below.
  837. // check to see if this featureloc already exists, but also keep track of the
  838. // last rank value
  839. $rank = -1;
  840. $exists = 0;
  841. $featureloc_sql = "SELECT FL.featureloc_id,FL.fmin,FL.fmax,F.uniquename as srcname
  842. FROM {featureloc} FL
  843. INNER JOIN {feature} F on F.feature_id = FL.srcfeature_id
  844. WHERE FL.feature_id = %d
  845. ORDER BY rank ASC";
  846. $recs = db_query($featureloc_sql,$feature->feature_id);
  847. while ($featureloc = db_fetch_object($recs)){
  848. if(strcmp($featureloc->srcname,$landmark)==0 and
  849. $featureloc->fmin == $fmin and $featureloc->fmax == $fmax){
  850. // this is the same featureloc, so do nothing... no need to update
  851. //TODO: need more checks here
  852. print " No change to featureloc\n";
  853. $exists = 1;
  854. }
  855. $rank = $featureloc->rank;
  856. }
  857. if(!$exists){
  858. $rank++;
  859. // this feature location is new so add it
  860. if(!$phase){
  861. $phase = 'NULL';
  862. }
  863. if(strcmp($is_fmin_partial,'f')==0){
  864. $is_fmin_partial = 'false';
  865. }
  866. elseif(strcmp($is_fmin_partial,'t')==0){
  867. $is_fmin_partial = 'true';
  868. }
  869. if(strcmp($is_fmax_partial,'f')==0){
  870. $is_fmax_partial = 'false';
  871. }
  872. elseif(strcmp($is_fmax_partial,'t')==0){
  873. $is_fmax_partial = 'true';
  874. }
  875. print " Adding featureloc $srcfeature->uniquename fmin: $fmin, fmax: $fmax, strand: $strand, phase: $phase, rank: $rank\n";
  876. $fl_isql = "INSERT INTO {featureloc}
  877. (feature_id, srcfeature_id, fmin, is_fmin_partial, fmax, is_fmax_partial,
  878. strand, phase, residue_info, locgroup, rank)
  879. VALUES (%d,%d,%d,%s,%d,%s,%d,%s,'%s',%d,%d)";
  880. $result = db_query($fl_isql,$feature->feature_id,$srcfeature->feature_id,$fmin,$is_fmin_partial,$fmax,$is_fmax_partial,
  881. $strand,$phase,$residue_info,$locgroup,$rank);
  882. if(!$result){
  883. print "ERROR: failed to insert featureloc\n";
  884. exit;
  885. return 0;
  886. }
  887. }
  888. return 1;
  889. }